From 0d7cf0022d7bde729ba5dfa385d3a71a7ba0e851 Mon Sep 17 00:00:00 2001 From: Test User Date: Sat, 18 Jul 2026 01:42:40 +0000 Subject: [PATCH] Reapply "Simplify the supported backends: roll back the direct PostgreSQL and MySQL adapters" (#4857) This reverts commit 0cf3b3ca0924d4ef738bac4536ddad0536db756a. --- .github/workflows/conformance.yml | 63 +- .golangci.yml | 8 + CHANGELOG.md | 12 + PROPOSAL-pluggable-storage-backends.md | 17 +- beads.go | 16 +- beads_cgo.go | 24 +- beads_nocgo.go | 22 +- cmd/bd/backend_support.go | 45 + cmd/bd/bootstrap.go | 42 +- cmd/bd/bootstrap_backend_guard_test.go | 293 +++++++ cmd/bd/bootstrap_test.go | 41 +- cmd/bd/doctor.go | 122 ++- cmd/bd/doctor/deep.go | 11 +- cmd/bd/doctor/deep_test.go | 21 + cmd/bd/doctor/dolt_e2e_test.go | 2 +- cmd/bd/doctor/migration_validation.go | 6 +- cmd/bd/doctor/migration_validation_test.go | 31 +- cmd/bd/doctor/perf_dolt.go | 3 +- cmd/bd/doctor/perf_dolt_test.go | 24 + cmd/bd/doctor/server.go | 17 +- cmd/bd/doctor/server_backend_test.go | 30 + cmd/bd/doctor_backend_guard_test.go | 61 ++ cmd/bd/doctor_embedded_test.go | 26 + cmd/bd/doctor_validate_test.go | 2 +- cmd/bd/dolt.go | 111 +-- cmd/bd/init.go | 352 ++------ cmd/bd/init_embedded_test.go | 13 +- cmd/bd/init_test.go | 40 +- cmd/bd/main.go | 132 +-- cmd/bd/main_test.go | 6 + cmd/bd/store_factory.go | 19 +- cmd/bd/store_factory_nocgo.go | 18 +- .../store_selection_removed_backend_test.go | 403 +++++++++ cmd/bd/telemetry_redact.go | 129 +++ cmd/bd/telemetry_scrub_test.go | 262 +++++- cmd/bd/version_tracking.go | 4 + cmd/bd/version_tracking_test.go | 2 +- default.nix | 2 +- docs/architecture/dolt.md | 37 +- docs/cli-reference/doctor.md | 33 +- docs/getting-started/quickstart.md | 4 +- docs/multi-agent/federation.md | 3 +- docs/reference/configuration.md | 55 +- examples/library-usage/README.md | 10 +- examples/library-usage/main.go | 14 +- examples/library-usage/main_test.go | 10 +- go.mod | 4 - go.sum | 7 - internal/beads/beads.go | 35 +- internal/beads/sqlite_database_path_test.go | 85 ++ internal/configfile/backend_messages.go | 38 + internal/configfile/configfile.go | 116 +-- internal/configfile/configfile_test.go | 31 +- internal/configfile/mysql_dsn_test.go | 37 - internal/configfile/postgres_dsn_test.go | 36 - internal/storage/backend_marker.go | 7 +- .../audit_config_metadata_slots_repomtime.go | 5 +- .../audit_labels-comments-events.go | 2 +- .../audit_molecule_wisp_batch_iter.go | 5 +- .../conformance/audit_search-counts-stats.go | 10 +- internal/storage/conformance/claim.go | 104 +-- internal/storage/conformance/conformance.go | 11 +- internal/storage/conformance/portable.go | 8 +- internal/storage/dolt/open.go | 25 + internal/storage/dolt/open_test.go | 44 + internal/storage/embeddeddolt/iter_stubs.go | 4 +- internal/storage/errors_unsupported.go | 2 +- internal/storage/issueops/blocked_state.go | 23 - internal/storage/issueops/count.go | 4 +- internal/storage/issueops/create.go | 7 +- internal/storage/issueops/helpers.go | 7 +- internal/storage/iter.go | 2 +- internal/storage/mysql/completeness_test.go | 107 --- internal/storage/mysql/conformance_test.go | 80 -- internal/storage/mysql/credential.go | 67 -- internal/storage/mysql/credential_test.go | 288 ------- internal/storage/mysql/dialect.go | 23 - internal/storage/mysql/dsn.go | 53 -- internal/storage/mysql/fromconfig.go | 57 -- internal/storage/mysql/redact_test.go | 117 --- internal/storage/mysql/schema.go | 162 ---- internal/storage/mysql/schema.sql | 344 -------- internal/storage/mysql/seed_test.go | 54 -- internal/storage/mysql/store.go | 69 -- internal/storage/mysql/unsupported.go | 17 - internal/storage/mysql/unsupported_gen.go | 236 ------ internal/storage/mysqldialect/credential.go | 83 -- .../mysqldialect/credential_extra_test.go | 81 -- .../storage/mysqldialect/credential_test.go | 82 -- internal/storage/mysqldialect/driver.go | 91 -- internal/storage/mysqldialect/rewrite.go | 43 - internal/storage/mysqldialect/rewrite_test.go | 123 --- internal/storage/pgdialect/corpus_test.go | 430 ---------- internal/storage/pgdialect/credential.go | 121 --- .../pgdialect/credential_extra_test.go | 148 ---- internal/storage/pgdialect/credential_test.go | 96 --- internal/storage/pgdialect/driver.go | 190 ----- internal/storage/pgdialect/redact.go | 214 ----- .../storage/pgdialect/redact_error_test.go | 188 ----- .../storage/pgdialect/redact_matrix_test.go | 179 ---- internal/storage/pgdialect/redact_test.go | 58 -- internal/storage/pgdialect/translate.go | 778 ------------------ internal/storage/pgdialect/translate_test.go | 108 --- internal/storage/postgres/collation.go | 49 -- internal/storage/postgres/collation_test.go | 29 - .../storage/postgres/completeness_test.go | 107 --- internal/storage/postgres/conformance_test.go | 89 -- internal/storage/postgres/credential.go | 67 -- internal/storage/postgres/credential_test.go | 260 ------ internal/storage/postgres/dialect.go | 22 - internal/storage/postgres/fromconfig.go | 68 -- internal/storage/postgres/repro_instr_test.go | 46 -- internal/storage/postgres/schema.go | 685 --------------- internal/storage/postgres/schema_test.go | 155 ---- internal/storage/postgres/seed_test.go | 57 -- internal/storage/postgres/smoke_test.go | 104 --- internal/storage/postgres/store.go | 87 -- internal/storage/postgres/unsupported.go | 24 - internal/storage/postgres/unsupported_gen.go | 236 ------ internal/storage/sqlbuild/sort.go | 10 +- internal/storage/sqlite/completeness_test.go | 14 +- internal/storage/sqlite/dsn.go | 15 +- internal/storage/sqlite/schema.go | 2 +- internal/storage/sqlite/unsupported.go | 4 +- internal/storage/sqlitedialect/driver.go | 4 +- internal/storage/sqlitedialect/translate.go | 16 +- internal/storage/sqlkit/queries.go | 4 +- internal/storage/sqlkit/store.go | 6 +- internal/storage/sqlkit/strategy.go | 13 +- internal/storage/unsupportedgen/gen_test.go | 2 +- internal/storage/unsupportedgen/main.go | 6 +- internal/utils/id_parser.go | 8 +- open_backend_selection_test.go | 118 +++ scripts/conformance.sh | 69 +- scripts/run-oracle-p.sh | 56 -- test/conformance/e2e_test.go | 63 +- test/conformance/handle.go | 61 -- test/conformance/profiles.go | 55 +- test/conformance/roundtrip_test.go | 21 +- 139 files changed, 2435 insertions(+), 8141 deletions(-) create mode 100644 cmd/bd/backend_support.go create mode 100644 cmd/bd/bootstrap_backend_guard_test.go create mode 100644 cmd/bd/doctor/server_backend_test.go create mode 100644 cmd/bd/doctor_backend_guard_test.go create mode 100644 cmd/bd/store_selection_removed_backend_test.go create mode 100644 cmd/bd/telemetry_redact.go create mode 100644 internal/beads/sqlite_database_path_test.go create mode 100644 internal/configfile/backend_messages.go delete mode 100644 internal/configfile/mysql_dsn_test.go delete mode 100644 internal/configfile/postgres_dsn_test.go delete mode 100644 internal/storage/mysql/completeness_test.go delete mode 100644 internal/storage/mysql/conformance_test.go delete mode 100644 internal/storage/mysql/credential.go delete mode 100644 internal/storage/mysql/credential_test.go delete mode 100644 internal/storage/mysql/dialect.go delete mode 100644 internal/storage/mysql/dsn.go delete mode 100644 internal/storage/mysql/fromconfig.go delete mode 100644 internal/storage/mysql/redact_test.go delete mode 100644 internal/storage/mysql/schema.go delete mode 100644 internal/storage/mysql/schema.sql delete mode 100644 internal/storage/mysql/seed_test.go delete mode 100644 internal/storage/mysql/store.go delete mode 100644 internal/storage/mysql/unsupported.go delete mode 100644 internal/storage/mysql/unsupported_gen.go delete mode 100644 internal/storage/mysqldialect/credential.go delete mode 100644 internal/storage/mysqldialect/credential_extra_test.go delete mode 100644 internal/storage/mysqldialect/credential_test.go delete mode 100644 internal/storage/mysqldialect/driver.go delete mode 100644 internal/storage/mysqldialect/rewrite.go delete mode 100644 internal/storage/mysqldialect/rewrite_test.go delete mode 100644 internal/storage/pgdialect/corpus_test.go delete mode 100644 internal/storage/pgdialect/credential.go delete mode 100644 internal/storage/pgdialect/credential_extra_test.go delete mode 100644 internal/storage/pgdialect/credential_test.go delete mode 100644 internal/storage/pgdialect/driver.go delete mode 100644 internal/storage/pgdialect/redact.go delete mode 100644 internal/storage/pgdialect/redact_error_test.go delete mode 100644 internal/storage/pgdialect/redact_matrix_test.go delete mode 100644 internal/storage/pgdialect/redact_test.go delete mode 100644 internal/storage/pgdialect/translate.go delete mode 100644 internal/storage/pgdialect/translate_test.go delete mode 100644 internal/storage/postgres/collation.go delete mode 100644 internal/storage/postgres/collation_test.go delete mode 100644 internal/storage/postgres/completeness_test.go delete mode 100644 internal/storage/postgres/conformance_test.go delete mode 100644 internal/storage/postgres/credential.go delete mode 100644 internal/storage/postgres/credential_test.go delete mode 100644 internal/storage/postgres/dialect.go delete mode 100644 internal/storage/postgres/fromconfig.go delete mode 100644 internal/storage/postgres/repro_instr_test.go delete mode 100644 internal/storage/postgres/schema.go delete mode 100644 internal/storage/postgres/schema_test.go delete mode 100644 internal/storage/postgres/seed_test.go delete mode 100644 internal/storage/postgres/smoke_test.go delete mode 100644 internal/storage/postgres/store.go delete mode 100644 internal/storage/postgres/unsupported.go delete mode 100644 internal/storage/postgres/unsupported_gen.go create mode 100644 open_backend_selection_test.go delete mode 100755 scripts/run-oracle-p.sh delete mode 100644 test/conformance/handle.go diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 76cfc9caf..9b0f55a73 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,12 +1,7 @@ name: Conformance -# Runs the two-tier storage-backend conformance suite against REAL databases — -# the same scripts/conformance.sh developers run locally, so local == CI. Tier 1 -# is the in-process store suite plus the wedge gates; Tier 2 is the end-to-end -# `bd init` + CLI differential (every SQL backend compared to the Dolt reference). -# Postgres and MySQL run as service containers; Dolt (embedded) and SQLite are -# in-process. The deep bts-rs 523-scenario oracle stays on-demand (CONFORMANCE_DEEP), -# not in this per-PR job. +# Runs the same two-tier Dolt/SQLite storage conformance suite developers run +# locally: the in-process store corpus and the real-binary CLI differential. on: pull_request: @@ -23,40 +18,8 @@ env: jobs: conformance: - name: Storage backend conformance (real DBs) + name: Storage backend conformance (Dolt + SQLite) runs-on: ubuntu-latest - services: - postgres: - image: postgres:16 - env: - POSTGRES_USER: bts - POSTGRES_PASSWORD: bts - POSTGRES_DB: bdwedge - # Code-point collation parity with Dolt (utf8mb4_0900_bin). The pg backend - # provisions per-workspace SCHEMAS inside this one database and can't set - # per-column COLLATE "C" (it breaks the shared recursive CTEs; see - # internal/storage/postgres/schema.go), so byte-order parity has to be - # guaranteed at DB-creation time via a C-locale cluster. - POSTGRES_INITDB_ARGS: "--lc-collate=C --lc-ctype=C" - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U bts -d bdwedge" - --health-interval 5s - --health-timeout 5s - --health-retries 20 - mysql: - image: mysql:8 - env: - MYSQL_ROOT_PASSWORD: bts - MYSQL_DATABASE: bdwedge - ports: - - 3306:3306 - options: >- - --health-cmd "mysqladmin ping -uroot -pbts --silent" - --health-interval 5s - --health-timeout 5s - --health-retries 30 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6 @@ -65,23 +28,5 @@ jobs: with: go-version-file: 'go.mod' - # The MySQL image's root only auths from localhost inside the container; the - # backend connects over TCP as a distinct host, so create a TCP-reachable user - # matching BEADS_MYSQL_TEST_URL. - - name: Provision MySQL user - run: | - mysql -h 127.0.0.1 -P 3306 -uroot -pbts -e " - CREATE USER IF NOT EXISTS 'bts'@'%' IDENTIFIED BY 'bts'; - GRANT ALL PRIVILEGES ON *.* TO 'bts'@'%' WITH GRANT OPTION; - FLUSH PRIVILEGES;" - - - name: Run conformance (Dolt + Postgres + MySQL + SQLite) - env: - BEADS_PG_TEST_URL: "postgres://bts:bts@127.0.0.1:5432/bdwedge" - BEADS_PG_PASSWORD: "bts" - BEADS_MYSQL_TEST_URL: "bts:bts@tcp(127.0.0.1:3306)/" - # Hard-fail (Tier 1 run_sql_conformance + Tier 2 e2e) if either real backend - # is absent, so removing/renaming a *_TEST_URL can't silently drop coverage - # while the gate still reports green. - BEADS_CONFORMANCE_REQUIRE: "postgres,mysql" + - name: Run conformance (Dolt + SQLite) run: ./scripts/conformance.sh diff --git a/.golangci.yml b/.golangci.yml index 9982d481c..9dfdfad6a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -158,6 +158,14 @@ linters: linters: - gosec text: "G705" + # G705: XSS via taint in CLI progress/warning/error output — every flagged + # sink is fmt.Fprint* to the terminal (os.Stderr or cobra's stdout/stderr + # writers), never a browser context (surfaced by the gosec bundled with + # golangci-lint v2.10) + - path: 'cmd/bd/(ado|doctor|errors|github|gitlab|hooks|jira|linear|main|notion|sync_push_pull)\.go' + linters: + - gosec + text: "G705" # errcheck: fmt.Fprintf errors are noise in CLI help text generation - path: 'cmd/bd/help_all\.go' linters: diff --git a/CHANGELOG.md b/CHANGELOG.md index b94004485..f6391c4e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Storage backend scope simplified** (bd-sadcd). The recently merged direct + PostgreSQL and MySQL adapters have been rolled back before entering a tagged + release. Supporting additional general-purpose server databases introduced + dialect, credential, schema-lifecycle, migration, CI, and operational + complexity at odds with our goal of keeping Beads as simple as possible and + consuming as few resources as possible. The storage interface, shared issue + core, SQLite implementation, and conformance harness remain; embedded Dolt, + Dolt server, and SQLite are the supported storage paths. Dolt server mode and + its use of the MySQL wire protocol are unchanged. Existing PostgreSQL or + MySQL workspaces stop before their configured databases are opened or + modified; see the [migration + guide](docs/architecture/storage-backends.md#existing-postgresql-or-mysql-workspaces). - **Cross-type blocking dependencies are now allowed** (bd-wg7ve, [#4034](https://github.com/gastownhall/beads/pull/4034)). `bd dep add ` — gating a work item on an epic (program) diff --git a/PROPOSAL-pluggable-storage-backends.md b/PROPOSAL-pluggable-storage-backends.md index 16e2edda3..0fc04a82d 100644 --- a/PROPOSAL-pluggable-storage-backends.md +++ b/PROPOSAL-pluggable-storage-backends.md @@ -1,6 +1,6 @@ # PROPOSAL: Pluggable Storage Backends -**Status:** Draft for review (survived one adversarial review round; defects folded in) +**Status:** Historical — partially implemented, then superseded on 2026-07-16 **Date:** 2026-07-02 **Informed by:** the bts-rs Rust spike (a from-scratch Rust reimplementation of the gc-contract surface that validated a backend-agnostic storage seam at 299/299 byte-parity @@ -8,6 +8,21 @@ across two radically different backends), a five-dimension recon of this repo's architecture, and a two-reviewer adversarial verification pass. All file:line citations verified against `worktree-beads-new`. +> **Superseded scope:** This proposal is retained as the design record for the +> storage interface, shared issue core, SQLite adapter, and differential +> conformance work that grew from it. Those pieces remain. The direct +> PostgreSQL and MySQL adapters were rolled back before entering a tagged +> release because supporting additional general-purpose server databases adds +> substantial dialect, credential, schema-lifecycle, migration, CI, and +> operational complexity. Our goal is to keep Beads as simple as possible and +> consume as few resources as possible. The supported paths are now embedded +> Dolt, Dolt server, and SQLite; Dolt server's use of the MySQL wire protocol is +> unchanged. See [Storage Backends](docs/architecture/storage-backends.md) for +> the current design. + +Everything below this notice is retained as historical design material, not as +current backend support or migration guidance. + --- ## 1. End state diff --git a/beads.go b/beads.go index 2976ce3d3..12f055a52 100644 --- a/beads.go +++ b/beads.go @@ -11,6 +11,7 @@ import ( "context" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/types" @@ -19,6 +20,15 @@ import ( // Storage is the interface for beads storage operations type Storage = beads.Storage +func configuredBackendUnavailable(backend string) error { + switch backend { + case configfile.BackendPostgres, configfile.BackendMySQL: + return configfile.RemovedBackendError(backend) + default: + return configfile.UnknownBackendError(backend) + } +} + // Transaction provides atomic multi-operation support within a database transaction. // Use Storage.RunInTransaction() to obtain a Transaction instance. type Transaction = beads.Transaction @@ -64,9 +74,9 @@ func Open(ctx context.Context, dbPath string) (Storage, error) { return dolt.New(ctx, &dolt.Config{Path: dbPath, CreateIfMissing: true}) } -// OpenFromConfig opens a beads database using configuration from metadata.json. -// Unlike Open, this respects Dolt server mode settings and database name -// configuration, connecting to the Dolt SQL server when dolt_mode is "server". +// OpenFromConfig opens the Dolt implementation using configuration from +// metadata.json. Unlike Open, this respects Dolt server mode settings and database +// name configuration. Use OpenBestAvailable when metadata may select SQLite. // beadsDir is the path to the .beads directory. func OpenFromConfig(ctx context.Context, beadsDir string) (Storage, error) { return dolt.NewFromConfigWithOptions(ctx, beadsDir, &dolt.Config{CreateIfMissing: true}) diff --git a/beads_cgo.go b/beads_cgo.go index a5736e73c..4bcc3213d 100644 --- a/beads_cgo.go +++ b/beads_cgo.go @@ -4,25 +4,43 @@ package beads import ( "context" + "fmt" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/storage/embeddeddolt" + beadssqlite "github.com/steveyegge/beads/internal/storage/sqlite" ) // OpenBestAvailable opens a beads database using the best available backend // for the given .beads directory. It reads metadata.json to determine the // configured mode: // -// - Embedded mode (default): Opens via the CGo embedded Dolt engine. -// - Server mode: Connects to an external dolt sql-server via OpenFromConfig. +// - Embedded Dolt (default): Opens via the CGo embedded Dolt engine. +// - Dolt server: Connects to a dolt sql-server via OpenFromConfig. +// - SQLite: Opens the configured local SQLite database. // // The returned Storage must be closed when no longer needed. // // beadsDir is the path to the .beads directory. func OpenBestAvailable(ctx context.Context, beadsDir string) (Storage, error) { cfg, err := configfile.Load(beadsDir) - if err == nil && cfg != nil && cfg.IsDoltServerMode() { + if err != nil { + return nil, fmt.Errorf("loading storage metadata: %w", err) + } + if cfg == nil { + cfg = configfile.DefaultConfig() + } + if !configfile.IsSupportedBackend(cfg.Backend) { + return nil, configuredBackendUnavailable(cfg.Backend) + } + + switch cfg.GetBackend() { + case configfile.BackendSQLite: + return beadssqlite.NewFromConfig(ctx, beadsDir) + } + + if cfg.IsDoltServerMode() { store, err := dolt.NewFromConfig(ctx, beadsDir) if err != nil { return nil, err diff --git a/beads_nocgo.go b/beads_nocgo.go index 58ebdab77..d2dcebc16 100644 --- a/beads_nocgo.go +++ b/beads_nocgo.go @@ -8,16 +8,32 @@ import ( "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/dolt" + beadssqlite "github.com/steveyegge/beads/internal/storage/sqlite" ) // OpenBestAvailable opens a beads database using the best available backend -// for the given .beads directory. In non-CGO builds, only server mode is -// supported; embedded mode returns an error directing the user to server mode. +// for the given .beads directory. In non-CGO builds, Dolt server and SQLite are +// supported; embedded Dolt returns an error directing the user to server mode. // // beadsDir is the path to the .beads directory. func OpenBestAvailable(ctx context.Context, beadsDir string) (Storage, error) { cfg, err := configfile.Load(beadsDir) - if err == nil && cfg != nil && cfg.IsDoltServerMode() { + if err != nil { + return nil, fmt.Errorf("loading storage metadata: %w", err) + } + if cfg == nil { + cfg = configfile.DefaultConfig() + } + if !configfile.IsSupportedBackend(cfg.Backend) { + return nil, configuredBackendUnavailable(cfg.Backend) + } + + switch cfg.GetBackend() { + case configfile.BackendSQLite: + return beadssqlite.NewFromConfig(ctx, beadsDir) + } + + if cfg.IsDoltServerMode() { store, err := dolt.NewFromConfig(ctx, beadsDir) if err != nil { return nil, err diff --git a/cmd/bd/backend_support.go b/cmd/bd/backend_support.go new file mode 100644 index 000000000..3da2a2de6 --- /dev/null +++ b/cmd/bd/backend_support.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + + "github.com/steveyegge/beads/internal/configfile" +) + +func validateConfiguredBackend(cfg *configfile.Config) error { + if cfg == nil { + return nil + } + switch cfg.Backend { + case configfile.BackendPostgres, configfile.BackendMySQL: + return configfile.RemovedBackendError(cfg.Backend) + case "", configfile.BackendDolt, configfile.BackendSQLite: + return nil + default: + return configfile.UnknownBackendError(cfg.Backend) + } +} + +func requireDoltBackend(cfg *configfile.Config) error { + if err := validateConfiguredBackend(cfg); err != nil { + return err + } + if cfg != nil && cfg.GetBackend() != configfile.BackendDolt { + return fmt.Errorf("not using Dolt backend (configured backend %q)", cfg.GetBackend()) + } + return nil +} + +func loadDoltBackendConfig(beadsDir string) (*configfile.Config, error) { + cfg, err := configfile.Load(beadsDir) + if err != nil { + return nil, fmt.Errorf("loading config: %w", err) + } + if cfg == nil { + cfg = configfile.DefaultConfig() + } + if err := requireDoltBackend(cfg); err != nil { + return nil, err + } + return cfg, nil +} diff --git a/cmd/bd/bootstrap.go b/cmd/bd/bootstrap.go index 3d0db9667..5e8521ed5 100644 --- a/cmd/bd/bootstrap.go +++ b/cmd/bd/bootstrap.go @@ -191,12 +191,21 @@ Examples: // name (e.g. dolt_database). Without this, server-mode rigs get the // default name "beads" instead of their configured name. (GH#3029) cfg, err := configfile.Load(beadsDir) - if err != nil || cfg == nil { - cfg = findParentConfig(beadsDir) + if err != nil { + return HandleError("failed to load %s: %v; no storage database was opened or modified; fix or restore metadata.json and retry", configfile.ConfigPath(beadsDir), err) + } + if cfg == nil { + cfg, err = findParentConfig(beadsDir) + if err != nil { + return HandleError("failed to load ancestor storage metadata: %v; no storage database was opened or modified; fix or restore metadata.json and retry", err) + } } if cfg == nil { cfg = configfile.DefaultConfig() } + if err := requireBootstrapDoltBackend(cfg); err != nil { + return HandleError("%v", err) + } resolvedCfg, repairMsg, err := applyBootstrapMetadataRepair(beadsDir, cfg, !dryRun) if err != nil { @@ -270,6 +279,16 @@ func noWorkspaceBootstrapPayload() map[string]interface{} { } } +func requireBootstrapDoltBackend(cfg *configfile.Config) error { + if err := validateConfiguredBackend(cfg); err != nil { + return err + } + if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { + return fmt.Errorf("bd bootstrap is not supported for SQLite workspaces; the configured SQLite database was not modified; run bd commands normally for this workspace") + } + return nil +} + func detectBootstrapAction(beadsDir string, cfg *configfile.Config) BootstrapPlan { plan := BootstrapPlan{ BeadsDir: beadsDir, @@ -513,6 +532,9 @@ func confirmPrompt(message string, nonInteractive bool) bool { } func executeBootstrapPlan(plan BootstrapPlan, cfg *configfile.Config, nonInteractive bool) error { + if err := requireBootstrapDoltBackend(cfg); err != nil { + return err + } if !confirmPrompt("Proceed?", nonInteractive) { fmt.Fprintf(os.Stderr, "Aborted.\n") return nil @@ -955,8 +977,10 @@ func isNonInteractiveBootstrap(flagValue bool) bool { // findParentConfig walks up from beadsDir's parent looking for a // .beads/metadata.json in ancestor directories. This handles the case where a // rig subdirectory (its own git repo) doesn't have a local .beads but its -// parent workspace does. Returns nil if no parent config is found. -func findParentConfig(beadsDir string) *configfile.Config { +// parent workspace does. Returns nil if no parent config is found. A malformed +// or unreadable ancestor metadata file is authoritative and returned as an error; +// bootstrap must not skip it and select a more distant workspace or defaults. +func findParentConfig(beadsDir string) (*configfile.Config, error) { // Start from the parent of beadsDir's enclosing directory. // beadsDir is typically "/.beads", so we start from 's parent. start := filepath.Dir(filepath.Dir(beadsDir)) @@ -964,8 +988,12 @@ func findParentConfig(beadsDir string) *configfile.Config { for dir := start; dir != "/" && dir != "."; { candidate := filepath.Join(dir, ".beads") - if cfg, err := configfile.Load(candidate); err == nil && cfg != nil { - return cfg + cfg, err := configfile.Load(candidate) + if err != nil { + return nil, fmt.Errorf("%s: %w", configfile.ConfigPath(candidate), err) + } + if cfg != nil { + return cfg, nil } // Don't search above $HOME @@ -979,7 +1007,7 @@ func findParentConfig(beadsDir string) *configfile.Config { } dir = parent } - return nil + return nil, nil } func init() { diff --git a/cmd/bd/bootstrap_backend_guard_test.go b/cmd/bd/bootstrap_backend_guard_test.go new file mode 100644 index 000000000..9988f4560 --- /dev/null +++ b/cmd/bd/bootstrap_backend_guard_test.go @@ -0,0 +1,293 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/configfile" +) + +func TestBootstrapRejectsRemovedBackendsBeforeWorkspaceWrites(t *testing.T) { + bd := buildBDForInitTests(t) + + for _, backend := range []string{configfile.BackendPostgres, configfile.BackendMySQL} { + backend := backend + for _, args := range [][]string{{"bootstrap", "--dry-run"}, {"bootstrap", "--yes"}} { + args := args + mode := "execute" + if len(args) > 1 && args[1] == "--dry-run" { + mode = "dry-run" + } + + t.Run(backend+"/"+mode, func(t *testing.T) { + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create .beads: %v", err) + } + + metadata := []byte(fmt.Sprintf("{\n \"database\": \"legacy.db\",\n \"backend\": %q,\n \"project_id\": \"legacy-project\"\n}\n", backend)) + metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) + if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + + cmd := exec.Command(bd, args...) + cmd.Dir = root + cmd.Env = bootstrapBackendGuardEnv(root, beadsDir) + out, err := cmd.CombinedOutput() + if err == nil { + t.Errorf("bd %s unexpectedly succeeded for removed backend %q:\n%s", strings.Join(args, " "), backend, out) + } + + message := strings.ToLower(string(out)) + for _, want := range []string{ + "no longer supported", + "general-purpose server databases", + "simple and resource-light", + "was not opened", + "export", + "dolt", + "sqlite", + } { + if !strings.Contains(message, want) { + t.Errorf("bd %s error for %q missing %q:\n%s", strings.Join(args, " "), backend, want, message) + } + } + + after, readErr := os.ReadFile(metadataPath) + if readErr != nil { + t.Fatalf("read metadata.json after rejected bootstrap: %v", readErr) + } + if !bytes.Equal(after, metadata) { + t.Errorf("bd %s rewrote metadata.json for removed backend %q:\nbefore:\n%s\nafter:\n%s", strings.Join(args, " "), backend, metadata, after) + } + assertNoBootstrapStorageArtifacts(t, beadsDir) + }) + } + } +} + +func TestBootstrapRejectsUnknownBackendBeforeWorkspaceWrites(t *testing.T) { + bd := buildBDForInitTests(t) + + for _, args := range [][]string{{"bootstrap", "--dry-run"}, {"bootstrap", "--yes"}} { + mode := "execute" + if args[1] == "--dry-run" { + mode = "dry-run" + } + + t.Run(mode, func(t *testing.T) { + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create .beads: %v", err) + } + + metadata := []byte("{\n \"backend\": \"mystery\",\n \"project_id\": \"legacy-project\"\n}\n") + metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) + if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + + cmd := exec.Command(bd, args...) + cmd.Dir = root + cmd.Env = bootstrapBackendGuardEnv(root, beadsDir) + out, err := cmd.CombinedOutput() + if err == nil { + t.Errorf("bd %s unexpectedly accepted an unknown backend:\n%s", strings.Join(args, " "), out) + } + message := string(out) + for _, want := range []string{"not recognized", "no storage database was opened or modified", "dolt", "sqlite"} { + if !strings.Contains(message, want) { + t.Errorf("bd %s error missing %q:\n%s", strings.Join(args, " "), want, message) + } + } + + after, readErr := os.ReadFile(metadataPath) + if readErr != nil { + t.Fatalf("read metadata.json after rejected bootstrap: %v", readErr) + } + if !bytes.Equal(after, metadata) { + t.Errorf("bd %s rewrote unknown-backend metadata:\nbefore:\n%s\nafter:\n%s", strings.Join(args, " "), metadata, after) + } + assertNoBootstrapStorageArtifacts(t, beadsDir) + }) + } +} + +func TestBootstrapRejectsCorruptMetadataBeforeWorkspaceWrites(t *testing.T) { + bd := buildBDForInitTests(t) + + for _, args := range [][]string{{"bootstrap", "--dry-run"}, {"bootstrap", "--yes"}} { + mode := "execute" + if args[1] == "--dry-run" { + mode = "dry-run" + } + + t.Run(mode, func(t *testing.T) { + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create .beads: %v", err) + } + + metadata := []byte("{\n") + metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) + if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { + t.Fatalf("write corrupt metadata.json: %v", err) + } + + cmd := exec.Command(bd, args...) + cmd.Dir = root + cmd.Env = bootstrapBackendGuardEnv(root, beadsDir) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("bd %s unexpectedly ignored corrupt metadata:\n%s", strings.Join(args, " "), out) + } + message := strings.ToLower(string(out)) + for _, want := range []string{"metadata.json", "no storage database was opened or modified"} { + if !strings.Contains(message, want) { + t.Errorf("bd %s error missing %q:\n%s", strings.Join(args, " "), want, out) + } + } + + after, readErr := os.ReadFile(metadataPath) + if readErr != nil { + t.Fatalf("read corrupt metadata after rejected bootstrap: %v", readErr) + } + if !bytes.Equal(after, metadata) { + t.Errorf("bd %s rewrote corrupt metadata:\nbefore: %q\nafter: %q", strings.Join(args, " "), metadata, after) + } + assertNoBootstrapStorageArtifacts(t, beadsDir) + }) + } +} + +func TestBootstrapDoesNotConvertExistingSQLiteWorkspace(t *testing.T) { + bd := buildBDForInitTests(t) + + for _, args := range [][]string{{"bootstrap", "--dry-run"}, {"bootstrap", "--yes"}} { + args := args + mode := "execute" + if len(args) > 1 && args[1] == "--dry-run" { + mode = "dry-run" + } + + t.Run(mode, func(t *testing.T) { + root := t.TempDir() + for _, gitArgs := range [][]string{ + {"init", "-q"}, + {"config", "user.email", "test@test.com"}, + {"config", "user.name", "Test"}, + {"config", "core.hooksPath", ".git/hooks"}, + } { + runGitForBootstrapTest(t, root, gitArgs...) + } + cmd := exec.Command(bd, "init", "--quiet", "--backend=sqlite", "--prefix=sqliteguard") + cmd.Dir = root + cmd.Env = bootstrapBackendGuardEnv(root, filepath.Join(root, ".beads")) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("initialize SQLite workspace: %v\n%s", err, out) + } + + beadsDir := filepath.Join(root, ".beads") + metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) + databasePath := filepath.Join(beadsDir, "beads.db") + metadataBefore := readBootstrapGuardFile(t, metadataPath) + databaseBefore := sha256.Sum256(readBootstrapGuardFile(t, databasePath)) + localVersionBefore, hadLocalVersion := readOptionalBootstrapGuardFile(t, filepath.Join(beadsDir, ".local_version")) + + cmd = exec.Command(bd, args...) + cmd.Dir = root + cmd.Env = bootstrapBackendGuardEnv(root, beadsDir) + out, err := cmd.CombinedOutput() + if err == nil { + t.Errorf("bd %s unexpectedly accepted an existing SQLite workspace:\n%s", strings.Join(args, " "), out) + } + + message := strings.ToLower(string(out)) + if !strings.Contains(message, "bootstrap") || !strings.Contains(message, "sqlite") || + !(strings.Contains(message, "not support") || strings.Contains(message, "unsupported") || + strings.Contains(message, "cannot") || strings.Contains(message, "refus")) { + t.Errorf("bd %s should fail with explicit SQLite bootstrap guidance:\n%s", strings.Join(args, " "), out) + } + + metadataAfter := readBootstrapGuardFile(t, metadataPath) + if !bytes.Equal(metadataAfter, metadataBefore) { + t.Errorf("bd %s rewrote SQLite metadata.json:\nbefore:\n%s\nafter:\n%s", strings.Join(args, " "), metadataBefore, metadataAfter) + } + databaseAfter := sha256.Sum256(readBootstrapGuardFile(t, databasePath)) + if databaseAfter != databaseBefore { + t.Errorf("bd %s modified the existing SQLite database", strings.Join(args, " ")) + } + localVersionAfter, hasLocalVersion := readOptionalBootstrapGuardFile(t, filepath.Join(beadsDir, ".local_version")) + if hasLocalVersion != hadLocalVersion || !bytes.Equal(localVersionAfter, localVersionBefore) { + t.Errorf("bd %s created or modified SQLite workspace version tracking", strings.Join(args, " ")) + } + for _, name := range []string{"embeddeddolt", "dolt"} { + path := filepath.Join(beadsDir, name) + if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { + t.Errorf("bd %s converted SQLite by creating Dolt state at %s (stat error: %v)", strings.Join(args, " "), path, statErr) + } + } + }) + } +} + +func bootstrapBackendGuardEnv(home, beadsDir string) []string { + env := make([]string, 0, len(os.Environ())+7) + for _, entry := range os.Environ() { + if strings.HasPrefix(entry, "BEADS_") || strings.HasPrefix(entry, "BD_") || strings.HasPrefix(entry, "HOME=") { + continue + } + env = append(env, entry) + } + return append(env, + "HOME="+home, + "BEADS_DIR="+beadsDir, + "BEADS_DOLT_AUTO_START=0", + "BEADS_NO_DAEMON=1", + "BEADS_TEST_IGNORE_REPO_CONFIG=1", + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + "BD_NON_INTERACTIVE=1", + ) +} + +func assertNoBootstrapStorageArtifacts(t *testing.T, beadsDir string) { + t.Helper() + for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version"} { + path := filepath.Join(beadsDir, name) + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("rejected bootstrap created local state at %s (stat error: %v)", path, err) + } + } +} + +func readBootstrapGuardFile(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return data +} + +func readOptionalBootstrapGuardFile(t *testing.T, path string) ([]byte, bool) { + t.Helper() + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return nil, false + } + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return data, true +} diff --git a/cmd/bd/bootstrap_test.go b/cmd/bd/bootstrap_test.go index 92be0d074..f274caed2 100644 --- a/cmd/bd/bootstrap_test.go +++ b/cmd/bd/bootstrap_test.go @@ -919,7 +919,10 @@ func TestBootstrapRigSubdirUsesParentDBName(t *testing.T) { cfg, cfgErr := configfile.Load(synthesizedDir) if cfgErr != nil || cfg == nil { // This is the fix path: search parent directories for metadata.json - cfg = findParentConfig(synthesizedDir) + cfg, cfgErr = findParentConfig(synthesizedDir) + if cfgErr != nil { + t.Fatalf("findParentConfig: %v", cfgErr) + } } if cfg == nil { cfg = configfile.DefaultConfig() @@ -939,6 +942,37 @@ func TestBootstrapRigSubdirUsesParentDBName(t *testing.T) { } } +func TestFindParentConfigDoesNotSkipCorruptNearestAncestor(t *testing.T) { + root := t.TempDir() + higherBeadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(higherBeadsDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(higherBeadsDir, "metadata.json"), []byte(`{"dolt_database":"wrong_higher_database"}`), 0o600); err != nil { + t.Fatal(err) + } + + nearestBeadsDir := filepath.Join(root, "workspace", ".beads") + if err := os.MkdirAll(nearestBeadsDir, 0o750); err != nil { + t.Fatal(err) + } + corrupt := []byte("{\n") + metadataPath := filepath.Join(nearestBeadsDir, "metadata.json") + if err := os.WriteFile(metadataPath, corrupt, 0o600); err != nil { + t.Fatal(err) + } + + synthesizedDir := filepath.Join(root, "workspace", "rig", ".beads") + if cfg, err := findParentConfig(synthesizedDir); err == nil { + t.Fatalf("findParentConfig skipped corrupt nearest metadata and selected higher database %q", cfg.GetDoltDatabase()) + } else if !strings.Contains(err.Error(), filepath.Join("workspace", ".beads", "metadata.json")) { + t.Fatalf("findParentConfig error = %v, want nearest metadata path", err) + } + if after, err := os.ReadFile(metadataPath); err != nil || string(after) != string(corrupt) { + t.Fatalf("corrupt nearest metadata changed: got %q, err %v", after, err) + } +} + // TestDetectBootstrapAction_SharedServerEnvUsesSharedPath verifies that when // BEADS_DOLT_SHARED_SERVER=1 is set but cfg.DoltMode is the default (embedded), // detectBootstrapAction looks in the shared-server directory — not embeddeddolt/. @@ -1052,7 +1086,10 @@ func TestDetectBootstrapAction_WorktreeSynthesizedDirPrefersSyncOverDefaultShare synthesizedDir := filepath.Join(localBare, ".beads") cfg, cfgErr := configfile.Load(synthesizedDir) if cfgErr != nil || cfg == nil { - cfg = findParentConfig(synthesizedDir) + cfg, cfgErr = findParentConfig(synthesizedDir) + if cfgErr != nil { + t.Fatalf("findParentConfig: %v", cfgErr) + } } if cfg == nil { cfg = configfile.DefaultConfig() diff --git a/cmd/bd/doctor.go b/cmd/bd/doctor.go index 43bb0e5b3..32cc3a88a 100644 --- a/cmd/bd/doctor.go +++ b/cmd/bd/doctor.go @@ -84,6 +84,13 @@ This command checks: - .beads/.gitignore up to date - Metadata.json version tracking (LastBdVersion field) +Storage Availability: + Full diagnostics, --perf, --deep, --server, --migration, and + --check=validate currently require Dolt server mode. Embedded Dolt and + SQLite support --check=artifacts, --check=conventions, and + --check=pollution. --check-health has a limited hook-health fallback. + Unsupported combinations return a notice without changing storage. + Performance Mode (--perf): Run performance diagnostics on your database: - Times key operations (bd ready, bd list, bd show, etc.) @@ -123,18 +130,12 @@ Server Mode (--server): - Schema compatible: Can query beads tables? - Connection pool: Pool health metrics -Migration Validation Mode (--migration): - Run Dolt migration validation checks with machine-parseable output. - Use --migration=pre before migration to verify readiness: - - JSONL file exists and is valid (parseable, no corruption) - - All JSONL issues are present in SQLite (or explains discrepancies) - - No blocking issues prevent migration - Use --migration=post after migration to verify completion: - - Dolt database exists and is healthy - - All issues from JSONL are present in Dolt - - No data was lost during migration - - Dolt database has no locks or uncommitted changes - Combine with --json for machine-parseable output for automation. +Legacy Dolt Migration Validation Mode (--migration): + Retained for older SQLite-to-Dolt migration workflows and available only in + Dolt server mode. It does not inspect or migrate a current SQLite workspace, + and it is not the migration path for removed PostgreSQL or MySQL backends. + Use the storage-backends migration guide for current backend moves. Combine + with --json for machine-parseable diagnostic output. Agent Mode (--agent): Output diagnostics designed for AI agent consumption. Instead of terse @@ -181,9 +182,9 @@ Examples: bd doctor --check=validate --fix # Auto-fix data-integrity issues bd doctor --deep # Full graph integrity validation bd doctor --server # Dolt server mode health checks - bd doctor --migration=pre # Validate readiness for Dolt migration - bd doctor --migration=post # Validate Dolt migration completed - bd doctor --migration=pre --json # Machine-parseable migration validation`, + bd doctor --migration=pre # Legacy Dolt-server migration diagnostic + bd doctor --migration=post # Legacy Dolt-server completion diagnostic + bd doctor --migration=pre --json # Machine-parseable legacy diagnostic`, SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -194,11 +195,6 @@ Examples: } }() - if usesProxiedServer() { - fmt.Fprintln(os.Stderr, "Note: 'bd doctor' is not yet supported in proxied-server mode.") - return nil - } - var checkPath string if len(args) > 0 { checkPath = args[0] @@ -212,6 +208,15 @@ Examples: if err != nil { return HandleError("failed to resolve path: %v", err) } + if err := validateDoctorWorkspaceBackend(absPath); err != nil { + return HandleError("%v", err) + } + doctorBackend := configuredDoctorBackend(absPath) + + if usesProxiedServer() { + fmt.Fprintln(os.Stderr, "Note: 'bd doctor' is not yet supported in proxied-server mode.") + return nil + } if doctorFix && isOrchestratorRoot(absPath) { return HandleErrorWithHint( @@ -229,8 +234,8 @@ Examples: // --perf opens a live server-mode connection; route embedded users to // the structured stub instead of a hard connection error (GH#3597). if perfMode { - if isEmbeddedMode() { - printEmbeddedUnsupported("doctor --perf") + if doctorBackend == configfile.BackendSQLite || isEmbeddedMode() { + printDoctorUnsupported("doctor --perf", doctorBackend) return nil } if err := doctor.RunPerformanceDiagnostics(absPath); err != nil { @@ -251,8 +256,8 @@ Examples: case "pollution": return runPollutionCheck(absPath, doctorClean, doctorYes) case "validate": - if isEmbeddedMode() { - printEmbeddedUnsupported("doctor --check=validate") + if doctorBackend == configfile.BackendSQLite || isEmbeddedMode() { + printDoctorUnsupported("doctor --check=validate", doctorBackend) return nil } return runValidateCheck(absPath) @@ -267,8 +272,8 @@ Examples: // time, each human-vetted — do not lift this gate wholesale. Checks // that reach into the database layer stay server-gated until the // storage driver interface covers them (AGENTS.md "Storage Boundary"). - if isEmbeddedMode() { - printEmbeddedUnsupported("doctor") + if doctorBackend == configfile.BackendSQLite || isEmbeddedMode() { + printDoctorUnsupported("doctor", doctorBackend) return nil } @@ -340,7 +345,7 @@ func init() { doctorCmd.Flags().BoolVar(&doctorOrchestrator, "orchestrator", false, "Running in orchestrator multi-workspace mode (routes.jsonl is expected, higher duplicate tolerance)") doctorCmd.Flags().IntVar(&orchestratorDuplicatesThreshold, "orchestrator-duplicates-threshold", 1000, "Duplicate tolerance threshold for orchestrator mode (wisps are ephemeral)") doctorCmd.Flags().BoolVar(&doctorServer, "server", false, "Run Dolt server mode health checks (connectivity, version, schema)") - doctorCmd.Flags().StringVar(&doctorMigration, "migration", "", "Run Dolt migration validation: 'pre' (before migration) or 'post' (after migration)") + doctorCmd.Flags().StringVar(&doctorMigration, "migration", "", "Run legacy Dolt-server migration diagnostics: 'pre' or 'post'") doctorCmd.Flags().BoolVar(&doctorAgent, "agent", false, "Agent-facing diagnostic mode: rich context for AI agents (ZFC-compliant)") } @@ -348,6 +353,69 @@ func shouldSkipDoctorNetworkChecks() bool { return jsonOutput || !ui.IsTerminal() } +// validateDoctorWorkspaceBackend keeps doctor diagnostics read-only when metadata +// selects a removed or unknown implementation or cannot be parsed. Doctor contains +// direct diagnostic store paths and may run under shared-server mode, so corrupt +// metadata must be rejected before version tracking or any database check begins. +func validateDoctorWorkspaceBackend(path string) error { + beadsDir := doctor.ResolveBeadsDirForRepo(path) + cfg, err := configfile.Load(beadsDir) + if err != nil { + return fmt.Errorf("failed to load %s: %w; no storage database was opened or modified; fix or restore metadata.json and retry", configfile.ConfigPath(beadsDir), err) + } + return validateConfiguredBackend(cfg) +} + +func configuredDoctorBackend(path string) string { + beadsDir := doctor.ResolveBeadsDirForRepo(path) + cfg, err := configfile.Load(beadsDir) + if err != nil || cfg == nil { + return configfile.BackendDolt + } + return cfg.GetBackend() +} + +func printDoctorUnsupported(commandLabel, backend string) { + if backend == configfile.BackendSQLite { + printSQLiteDoctorUnsupported(commandLabel) + return + } + printEmbeddedUnsupported(commandLabel) +} + +func printSQLiteDoctorUnsupported(commandLabel string) { + hints := []string{ + "Run normal SQLite-backed bd commands as usual", + "Check bd version: bd version", + "Use bd doctor --check=artifacts, --check=conventions, or --check=pollution", + } + supported := []string{"artifacts", "conventions", "pollution"} + unsupported := []string{"validate"} + + if jsonOutput || doctorAgent { + payload := map[string]interface{}{ + "error": fmt.Sprintf("'bd %s' database checks are not yet supported for SQLite; normal SQLite commands remain supported", commandLabel), + "code": "sqlite_doctor_unsupported", + "unsupported": true, + "backend": configfile.BackendSQLite, + "command": commandLabel, + "checks_supported_for_sqlite": supported, + "checks_not_yet_supported_for_sqlite": unsupported, + "hints": hints, + } + encoder := json.NewEncoder(os.Stderr) + encoder.SetIndent("", " ") + _ = encoder.Encode(wrapWithSchemaVersion(payload)) + return + } + + fmt.Fprintf(os.Stderr, "Note: 'bd %s' database checks are not yet supported for SQLite; normal SQLite commands remain supported.\n\n", commandLabel) + fmt.Fprintln(os.Stderr, "Checks available for SQLite workspaces:") + fmt.Fprintln(os.Stderr, " • bd doctor --check=artifacts") + fmt.Fprintln(os.Stderr, " • bd doctor --check=conventions") + fmt.Fprintln(os.Stderr, " • bd doctor --check=pollution") +} + // printEmbeddedUnsupported reports that a doctor variant is not yet wired up // for embedded mode. Emits a structured payload to stderr when --json or // --agent is set so downstream tooling can detect the gap without parsing diff --git a/cmd/bd/doctor/deep.go b/cmd/bd/doctor/deep.go index 21f06fc07..0173260ad 100644 --- a/cmd/bd/doctor/deep.go +++ b/cmd/bd/doctor/deep.go @@ -40,12 +40,17 @@ func RunDeepValidation(path string) DeepValidationResult { } if backend != configfile.BackendDolt { + status := StatusWarning + message := fmt.Sprintf("N/A (deep validation requires Dolt; configured backend is %q)", backend) + if backend == configfile.BackendSQLite { + status = StatusOK + message = "N/A (SQLite is supported; deep validation currently requires Dolt server mode)" + } check := DoctorCheck{ Name: "Deep Validation", - Status: StatusWarning, - Message: "SQLite backend detected", + Status: status, + Message: message, Category: CategoryMaintenance, - Fix: "Run 'bd init' to set up Dolt backend", } result.AllChecks = append(result.AllChecks, check) return result diff --git a/cmd/bd/doctor/deep_test.go b/cmd/bd/doctor/deep_test.go index 1e7c36361..8a7f1425f 100644 --- a/cmd/bd/doctor/deep_test.go +++ b/cmd/bd/doctor/deep_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/types" ) @@ -45,6 +46,26 @@ func TestRunDeepValidation_EmptyBeadsDir(t *testing.T) { } } +func TestRunDeepValidationSQLiteIsNotAMigrationWarning(t *testing.T) { + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + if err := (&configfile.Config{Backend: configfile.BackendSQLite, SQLitePath: "beads.db"}).Save(beadsDir); err != nil { + t.Fatalf("save SQLite config: %v", err) + } + + result := RunDeepValidation(tmpDir) + if !result.OverallOK || len(result.AllChecks) != 1 { + t.Fatalf("SQLite deep-validation result = %#v, want one benign N/A check", result) + } + check := result.AllChecks[0] + if check.Status != StatusOK || !strings.Contains(check.Message, "SQLite") || check.Fix != "" { + t.Fatalf("SQLite deep-validation check = %#v, want supported-backend N/A without migration fix", check) + } +} + // TestCheckParentConsistency_OrphanedDeps verifies detection of orphaned parent-child deps func TestCheckParentConsistency_OrphanedDeps(t *testing.T) { store := newTestDoltStore(t, "bd") diff --git a/cmd/bd/doctor/dolt_e2e_test.go b/cmd/bd/doctor/dolt_e2e_test.go index 080323fd7..22d3f6831 100644 --- a/cmd/bd/doctor/dolt_e2e_test.go +++ b/cmd/bd/doctor/dolt_e2e_test.go @@ -274,7 +274,7 @@ func runBDDoctor(t *testing.T, bdPath, path string) (e2eDoctorResult, string, er return result, string(out), execErr } -// TestE2E_DoctorSQLiteBackend was removed: SQLite backend no longer exists. +// TestE2E_DoctorSQLiteBackend is covered by the backend-neutral doctor tests. // GetBackend() always returns "dolt" after the dolt-native cleanup (bd-yqpwy). // TestE2E_DoctorDoltBackendNoDB was removed: the embedded Dolt driver diff --git a/cmd/bd/doctor/migration_validation.go b/cmd/bd/doctor/migration_validation.go index 13acdecc5..7e682d13c 100644 --- a/cmd/bd/doctor/migration_validation.go +++ b/cmd/bd/doctor/migration_validation.go @@ -148,7 +148,7 @@ func CheckMigrationReadiness(path string) (DoctorCheck, MigrationValidationResul Status: status, Message: message, Detail: strings.Join(result.Warnings, "\n"), - Fix: "Run 'bd migrate dolt' to start migration", + Fix: "Follow 'bd help init-safety' to reinitialize with Dolt, then import and verify this JSONL export", Category: CategoryMaintenance, }, result } @@ -195,7 +195,7 @@ func CheckMigrationCompletion(path string) (DoctorCheck, MigrationValidationResu Status: StatusError, Message: "Not using Dolt backend", Detail: fmt.Sprintf("Current backend: %s", result.Backend), - Fix: "Run 'bd migrate dolt' to migrate to Dolt", + Fix: "Follow 'bd help init-safety' to reinitialize with Dolt, then import and verify the issue export", Category: CategoryMaintenance, }, result } @@ -302,7 +302,7 @@ func CheckMigrationCompletion(path string) (DoctorCheck, MigrationValidationResu Status: StatusError, Message: fmt.Sprintf("Migration incomplete: %d error(s)", len(result.Errors)), Detail: strings.Join(result.Errors, "\n"), - Fix: "Re-run 'bd migrate dolt' or check for data issues", + Fix: "Check the export/import results; follow 'bd help init-safety' before reinitializing again", Category: CategoryMaintenance, }, result } diff --git a/cmd/bd/doctor/migration_validation_test.go b/cmd/bd/doctor/migration_validation_test.go index 001b93bb0..45ca66829 100644 --- a/cmd/bd/doctor/migration_validation_test.go +++ b/cmd/bd/doctor/migration_validation_test.go @@ -6,6 +6,7 @@ import ( "context" "os" "path/filepath" + "strings" "testing" ) @@ -135,13 +136,14 @@ func TestCheckMigrationReadinessResult_NoJSONL(t *testing.T) { if err := os.MkdirAll(beadsDir, 0755); err != nil { t.Fatalf("failed to create .beads: %v", err) } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"sqlite","sqlite_path":"issues.db"}`), 0644); err != nil { + t.Fatalf("failed to create SQLite metadata: %v", err) + } check, _ := CheckMigrationReadiness(tmpDir) - // With Dolt-only backend, GetBackend defaults to BackendDolt, - // so migration readiness returns OK ("Already using Dolt backend") - if check.Status != StatusOK { - t.Errorf("status = %q, want %q (Dolt-only backend returns OK)", check.Status, StatusOK) + if check.Status != StatusError { + t.Errorf("status = %q, want %q", check.Status, StatusError) } } @@ -156,6 +158,9 @@ func TestCheckMigrationReadinessResult_ValidJSONL(t *testing.T) { if err := os.MkdirAll(beadsDir, 0755); err != nil { t.Fatalf("failed to create .beads: %v", err) } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"sqlite","sqlite_path":"issues.db"}`), 0644); err != nil { + t.Fatalf("failed to create SQLite metadata: %v", err) + } // Create valid JSONL jsonl := `{"id":"bd-001","title":"Test 1"}` + "\n" + `{"id":"bd-002","title":"Test 2"}` @@ -165,12 +170,15 @@ func TestCheckMigrationReadinessResult_ValidJSONL(t *testing.T) { check, _ := CheckMigrationReadiness(tmpDir) - // With Dolt-only backend, GetBackend defaults to BackendDolt, - // so migration readiness returns OK ("Already using Dolt backend") - // without validating JSONL (no migration needed) if check.Status == StatusError { t.Errorf("status = %q, did not want error", check.Status) } + if !strings.Contains(check.Fix, "bd help init-safety") { + t.Errorf("fix must point to safe reinitialization guidance, got %q", check.Fix) + } + if strings.Contains(check.Fix, "bd migrate dolt") { + t.Errorf("fix must not recommend removed backend-conversion command, got %q", check.Fix) + } } func TestCheckMigrationCompletionResult_NoBeadsDir(t *testing.T) { @@ -206,6 +214,9 @@ func TestCheckMigrationCompletionResult_NotDoltBackend(t *testing.T) { if err := os.MkdirAll(beadsDir, 0755); err != nil { t.Fatalf("failed to create .beads: %v", err) } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"sqlite","sqlite_path":"issues.db"}`), 0644); err != nil { + t.Fatalf("failed to create SQLite metadata: %v", err) + } // Create JSONL (no Dolt backend present) jsonl := `{"id":"bd-001","title":"Test 1"}` @@ -226,6 +237,12 @@ func TestCheckMigrationCompletionResult_NotDoltBackend(t *testing.T) { if len(result.Errors) == 0 { t.Error("expected errors in result") } + if !strings.Contains(check.Fix, "bd help init-safety") { + t.Errorf("fix must point to safe reinitialization guidance, got %q", check.Fix) + } + if strings.Contains(check.Fix, "bd migrate dolt") { + t.Errorf("fix must not recommend removed backend-conversion command, got %q", check.Fix) + } } func TestMigrationValidationResult_JSONSerialization(t *testing.T) { diff --git a/cmd/bd/doctor/perf_dolt.go b/cmd/bd/doctor/perf_dolt.go index fd236e41f..b28234ca4 100644 --- a/cmd/bd/doctor/perf_dolt.go +++ b/cmd/bd/doctor/perf_dolt.go @@ -49,7 +49,8 @@ func RunDoltPerformanceDiagnostics(path string, enableProfiling bool) (*DoltPerf // Verify this is a Dolt backend if !IsDoltBackend(beadsDir) { - return nil, fmt.Errorf("SQLite backend is no longer supported. Migrate to Dolt with 'bd migrate'") + backend, _ := getBackendAndBeadsDir(path) + return nil, fmt.Errorf("Dolt server performance diagnostics do not apply to the configured backend %q; SQLite remains supported for normal bd commands", backend) } metrics := &DoltPerfMetrics{ diff --git a/cmd/bd/doctor/perf_dolt_test.go b/cmd/bd/doctor/perf_dolt_test.go index 80ae52262..0ee8d556c 100644 --- a/cmd/bd/doctor/perf_dolt_test.go +++ b/cmd/bd/doctor/perf_dolt_test.go @@ -40,3 +40,27 @@ func TestRunDoltPerformanceDiagnostics_RequiresServer(t *testing.T) { t.Errorf("expected server/database error, got: %v", err) } } + +func TestRunDoltPerformanceDiagnosticsSQLiteIsSupportedButNotApplicable(t *testing.T) { + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + cfg := &configfile.Config{Backend: configfile.BackendSQLite, SQLitePath: "beads.db"} + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("save SQLite config: %v", err) + } + + _, err := RunDoltPerformanceDiagnostics(tmpDir, false) + if err == nil { + t.Fatal("expected Dolt-only diagnostics to decline a SQLite workspace") + } + message := err.Error() + if strings.Contains(message, "no longer supported") || strings.Contains(message, "Migrate") { + t.Fatalf("Dolt-only diagnostics gave hostile SQLite migration guidance: %v", err) + } + if !strings.Contains(message, "Dolt") || !strings.Contains(message, "SQLite") { + t.Fatalf("diagnostic error should explain the backend mismatch: %v", err) + } +} diff --git a/cmd/bd/doctor/server.go b/cmd/bd/doctor/server.go index 2a91ffa82..8661b0e08 100644 --- a/cmd/bd/doctor/server.go +++ b/cmd/bd/doctor/server.go @@ -58,15 +58,22 @@ func RunServerHealthChecks(path string) ServerHealthResult { // Check if Dolt backend is configured if cfg.GetBackend() != configfile.BackendDolt { + status := StatusWarning + message := fmt.Sprintf("Server checks require Dolt; configured backend is %q", cfg.GetBackend()) + detail := "Dolt server health checks do not apply to this backend" + if cfg.GetBackend() == configfile.BackendSQLite { + status = StatusOK + message = "N/A (SQLite is supported and does not use a Dolt server)" + detail = "Run bd commands normally for this SQLite workspace" + } result.Checks = append(result.Checks, DoctorCheck{ Name: "Server Config", - Status: StatusWarning, - Message: fmt.Sprintf("Backend is '%s', not Dolt", cfg.GetBackend()), - Detail: "Server mode health checks are only relevant for Dolt backend", - Fix: "Set backend: dolt in metadata.json to use Dolt server mode", + Status: status, + Message: message, + Detail: detail, Category: CategoryFederation, }) - result.OverallOK = false + result.OverallOK = status == StatusOK return result } diff --git a/cmd/bd/doctor/server_backend_test.go b/cmd/bd/doctor/server_backend_test.go new file mode 100644 index 000000000..650468973 --- /dev/null +++ b/cmd/bd/doctor/server_backend_test.go @@ -0,0 +1,30 @@ +package doctor + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/configfile" +) + +func TestRunServerHealthChecksSQLiteIsNotAMigrationWarning(t *testing.T) { + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + if err := (&configfile.Config{Backend: configfile.BackendSQLite, SQLitePath: "beads.db"}).Save(beadsDir); err != nil { + t.Fatalf("save SQLite config: %v", err) + } + + result := RunServerHealthChecks(tmpDir) + if !result.OverallOK || len(result.Checks) != 1 { + t.Fatalf("SQLite server-health result = %#v, want one benign N/A check", result) + } + check := result.Checks[0] + if check.Status != StatusOK || !strings.Contains(check.Message, "SQLite") || check.Fix != "" { + t.Fatalf("SQLite server-health check = %#v, want supported-backend N/A without metadata edit", check) + } +} diff --git a/cmd/bd/doctor_backend_guard_test.go b/cmd/bd/doctor_backend_guard_test.go new file mode 100644 index 000000000..3d2262151 --- /dev/null +++ b/cmd/bd/doctor_backend_guard_test.go @@ -0,0 +1,61 @@ +//go:build cgo + +package main + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestDoctorRejectsCorruptMetadataBeforeSharedServerChecks(t *testing.T) { + bd := buildBDUnderTest(t) + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create .beads: %v", err) + } + + metadata := []byte("{\n") + metadataPath := filepath.Join(beadsDir, "metadata.json") + if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { + t.Fatalf("write corrupt metadata.json: %v", err) + } + + cmd := exec.Command(bd, "doctor", "--json") + cmd.Dir = root + cmd.Env = append(removedBackendTestEnv(beadsDir), + "BEADS_DOLT_SHARED_SERVER=1", + "BEADS_DOLT_SERVER_PORT=1", + "BEADS_DOLT_AUTO_START=0", + "BEADS_NO_DAEMON=1", + "BD_DISABLE_METRICS=1", + ) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("doctor unexpectedly ran shared-server checks with corrupt metadata:\n%s", out) + } + + message := strings.ToLower(string(out)) + for _, want := range []string{"metadata.json", "parsing config", "no storage database was opened or modified"} { + if !strings.Contains(message, want) { + t.Errorf("doctor corrupt-metadata error missing %q:\n%s", want, out) + } + } + + after, readErr := os.ReadFile(metadataPath) + if readErr != nil { + t.Fatalf("read metadata after refusal: %v", readErr) + } + if !bytes.Equal(after, metadata) { + t.Fatalf("doctor rewrote corrupt metadata:\nbefore: %q\nafter: %q", metadata, after) + } + for _, name := range []string{".local_version", "embeddeddolt", "dolt", "beads.db", "dolt-server.pid", "dolt-server.port"} { + if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { + t.Fatalf("doctor created %s before rejecting corrupt metadata (stat error: %v)", name, statErr) + } + } +} diff --git a/cmd/bd/doctor_embedded_test.go b/cmd/bd/doctor_embedded_test.go index e4f340be6..da9963c97 100644 --- a/cmd/bd/doctor_embedded_test.go +++ b/cmd/bd/doctor_embedded_test.go @@ -83,6 +83,32 @@ func TestDoctorEmbeddedSupportedChecks(t *testing.T) { } } +func TestDoctorSQLiteUnsupportedMessagingDoesNotRecommendDoltMigration(t *testing.T) { + bd := buildBDForInitTests(t) + dir, _, _ := bdInit(t, bd, "--prefix", "ds", "--backend", "sqlite") + + stdout, stderr, err := runBDSplit(t, bd, dir, "doctor", "--json") + if err != nil { + t.Fatalf("bd doctor --json failed for SQLite: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if strings.TrimSpace(stdout) != "" { + t.Fatalf("expected empty stdout for SQLite doctor stub, got:\n%s", stdout) + } + + var result map[string]interface{} + if err := json.Unmarshal([]byte(stderr), &result); err != nil { + t.Fatalf("SQLite doctor stub is not valid JSON: %v\nstderr:\n%s", err, stderr) + } + if result["code"] != "sqlite_doctor_unsupported" || result["backend"] != "sqlite" { + t.Fatalf("SQLite doctor payload = %#v, want backend-specific unsupported result", result) + } + for _, hostile := range []string{"embeddeddolt", "Switch to server mode", "migrate to Dolt", "no longer supported"} { + if strings.Contains(stderr, hostile) { + t.Errorf("SQLite doctor guidance contains %q:\n%s", hostile, stderr) + } + } +} + // TestDoctorEmbeddedUnsupportedJSON verifies that variants which still require // server mode (bare doctor, --check=validate) emit a structured JSON payload // when --json or --agent --json is set, rather than the prose stub. Tooling diff --git a/cmd/bd/doctor_validate_test.go b/cmd/bd/doctor_validate_test.go index 5b6751006..5e3281cc4 100644 --- a/cmd/bd/doctor_validate_test.go +++ b/cmd/bd/doctor_validate_test.go @@ -144,7 +144,7 @@ func TestValidateCheck_DetectsOrphanedDeps(t *testing.T) { } func TestValidateCheck_GitConflicts_DoltClean(t *testing.T) { - // Since GetBackend() always returns "dolt" (SQLite removed in 87493ce9), + // When metadata is absent, GetBackend() defaults to "dolt", // the Git Conflicts check now queries dolt_conflicts (GH-2249). // Verify it reports OK for a clean Dolt database. tmpDir, store := setupValidateTestDB(t, "val") diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index 1551917c5..5334e0846 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -106,6 +106,13 @@ Examples: bd dolt set data-dir /home/user/.beads-dolt/myproject`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { + beadsDir := selectedDoltBeadsDir() + if beadsDir == "" { + return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) + } + if _, err := loadDoltBackendConfig(beadsDir); err != nil { + return HandleError("%v", err) + } if !usesSQLServer() { return HandleError("'bd dolt set' is not supported in embedded mode (no Dolt server)") } @@ -129,6 +136,13 @@ This verifies that: Use this before switching to server mode to ensure the server is running.`, RunE: func(cmd *cobra.Command, args []string) error { + beadsDir := selectedDoltBeadsDir() + if beadsDir == "" { + return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) + } + if _, err := loadDoltBackendConfig(beadsDir); err != nil { + return HandleError("%v", err) + } if !usesSQLServer() { return HandleError("'bd dolt test' is not supported in embedded mode (no Dolt server)") } @@ -489,12 +503,12 @@ The remote must already exist (see 'bd dolt remote add').`, } // errExplicitCommitUnsupported returns a typed *storage.ErrUnsupported when the -// open store has no Dolt commit graph (Postgres/MySQL/SQLite), and nil for Dolt. -// The SQL backends deliberately expose a no-op Commit so internal write paths can +// open store has no Dolt commit graph (currently SQLite), and nil for Dolt. +// Non-Dolt stores deliberately expose a no-op Commit so internal write paths can // call store.Commit() opportunistically without erroring; an EXPLICIT `bd dolt // commit` / `bd vc commit` must not ride that no-op and print a fake "Committed." // with no commit graph behind it. Internal opportunistic auto-commit is skipped -// separately in PostRun for these backends, so gating only the explicit commands +// separately in PostRun for these stores, so gating only the explicit commands // keeps that path untouched. func errExplicitCommitUnsupported(st storage.DoltStorage, op string) error { ncg, ok := storage.UnwrapStore(st).(storage.NonCommitGraphBackend) @@ -566,13 +580,16 @@ project path. PID and logs are stored in .beads/. The server auto-starts transparently when needed, so manual start is rarely required. Use this command for explicit control or diagnostics.`, RunE: func(cmd *cobra.Command, args []string) error { - if !usesSQLServer() { - return HandleError("'bd dolt start' is not supported in embedded mode (no Dolt server)") - } beadsDir := selectedDoltBeadsDir() if beadsDir == "" { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } + if _, err := loadDoltBackendConfig(beadsDir); err != nil { + return HandleError("%v", err) + } + if !usesSQLServer() { + return HandleError("'bd dolt start' is not supported in embedded mode (no Dolt server)") + } serverDir := doltserver.ResolveServerDir(beadsDir) state, err := doltserver.Start(serverDir) @@ -609,13 +626,16 @@ var doltStopCmd = &cobra.Command{ This sends a graceful shutdown signal. The server will restart automatically on the next bd command unless auto-start is disabled.`, RunE: func(cmd *cobra.Command, args []string) error { - if !usesSQLServer() { - return HandleError("'bd dolt stop' is not supported in embedded mode (no Dolt server)") - } beadsDir := selectedDoltBeadsDir() if beadsDir == "" { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } + if _, err := loadDoltBackendConfig(beadsDir); err != nil { + return HandleError("%v", err) + } + if !usesSQLServer() { + return HandleError("'bd dolt stop' is not supported in embedded mode (no Dolt server)") + } if usesProxiedServer() { rootDir, err := resolveProxiedServerRootPath(beadsDir) @@ -659,10 +679,20 @@ endpoint via SQL and reports reachability, server version, and database.`, if beadsDir == "" { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - // A non-Dolt backend (postgres/mysql/sqlite) has no Dolt engine at all; + cfg, cfgErr := configfile.Load(beadsDir) + if cfgErr != nil { + return HandleError("loading config: %v", cfgErr) + } + if cfg == nil { + cfg = configfile.DefaultConfig() + } + if err := validateConfiguredBackend(cfg); err != nil { + return HandleError("%v", err) + } + // A non-Dolt backend (SQLite or a removed-backend tombstone) has no Dolt engine; // report the backend rather than misdescribing an embedded Dolt server // (parity with `bd dolt show`, which already special-cases this). - if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil && cfg.GetBackend() != configfile.BackendDolt { + if cfg.GetBackend() != configfile.BackendDolt { fmt.Printf("Backend: %s (no Dolt engine)\n", cfg.GetBackend()) return nil } @@ -679,19 +709,10 @@ endpoint via SQL and reports reachability, server version, and database.`, // systemd manages the server lifecycle, be-0eyj) // // IsAutoStartDisabled reads the active (globally-bound) config and - // BEADS_DOLT_AUTO_START env, not the per-beadsDir cfg loaded just - // below. That coupling is intentional and consistent with every + // BEADS_DOLT_AUTO_START env, not the per-beadsDir cfg loaded above. + // That coupling is intentional and consistent with every // other call site of IsAutoStartDisabled in this package — both // resolve against the same active workspace at command time. - cfg, cfgErr := configfile.Load(beadsDir) - if cfgErr != nil { - // Don't silently swallow. A corrupted or missing metadata.json - // would otherwise mask the externally-managed routing for both - // the remote-host and auto-start-disabled-local cases, falling - // through to the PID-file path with a misleading "not running" - // — which is the exact failure mode this PR addresses. - fmt.Fprintf(os.Stderr, "Warning: cannot load .beads config (%v); falling back to PID-file status path\n", cfgErr) - } if cfg != nil && shouldUseExternalDoltStatus(cfg, doltserver.IsAutoStartDisabled(), doltserver.IsSharedServerMode()) { runExternalDoltStatus(beadsDir, cfg) return nil @@ -942,10 +963,15 @@ servers are preserved.`, SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { + beadsDir := selectedDoltBeadsDir() + if beadsDir != "" { + if _, err := loadDoltBackendConfig(beadsDir); err != nil { + return HandleError("%v", err) + } + } if !usesSQLServer() { return HandleError("'bd dolt killall' is not supported in embedded mode (no Dolt server)") } - beadsDir := selectedDoltBeadsDir() if beadsDir == "" { beadsDir = "." // best effort } @@ -1007,6 +1033,13 @@ Stale database prefixes: testdb_*, beads_test*, beads_pt*, beads_vr*, doctest_*, These waste server memory and can degrade performance under concurrent load. Use --dry-run to see what would be dropped without actually dropping.`, RunE: func(cmd *cobra.Command, args []string) error { + beadsDir := selectedDoltBeadsDir() + if beadsDir == "" { + return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) + } + if _, err := loadDoltBackendConfig(beadsDir); err != nil { + return HandleError("%v", err) + } if !usesSQLServer() { return HandleError("'bd dolt clean-databases' is not supported in embedded mode (no Dolt server)") } @@ -1450,6 +1483,9 @@ func showDoltConfig(testConnection bool) error { if cfg == nil { cfg = configfile.DefaultConfig() } + if err := validateConfiguredBackend(cfg); err != nil { + return HandleError("%v", err) + } backend := cfg.GetBackend() embedded := !usesSQLServer() @@ -1548,16 +1584,9 @@ func setDoltConfig(key, value string, updateConfig bool) error { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - cfg, err := configfile.Load(beadsDir) + cfg, err := loadDoltBackendConfig(beadsDir) if err != nil { - return HandleError("loading config: %v", err) - } - if cfg == nil { - cfg = configfile.DefaultConfig() - } - - if cfg.GetBackend() != configfile.BackendDolt { - return HandleError("not using Dolt backend") + return HandleError("%v", err) } var yamlKey string @@ -1709,16 +1738,9 @@ func testDoltConnection() error { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - cfg, err := configfile.Load(beadsDir) + cfg, err := loadDoltBackendConfig(beadsDir) if err != nil { - return HandleError("loading config: %v", err) - } - if cfg == nil { - cfg = configfile.DefaultConfig() - } - - if cfg.GetBackend() != configfile.BackendDolt { - return HandleError("not using Dolt backend") + return HandleError("%v", err) } host := cfg.GetDoltServerHost() @@ -1878,12 +1900,9 @@ func openDoltServerConnection() (*sql.DB, func(), error) { return nil, nil, HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - cfg, err := configfile.Load(beadsDir) + cfg, err := loadDoltBackendConfig(beadsDir) if err != nil { - return nil, nil, HandleError("loading config: %v", err) - } - if cfg == nil { - cfg = configfile.DefaultConfig() + return nil, nil, HandleError("%v", err) } host := cfg.GetDoltServerHost() diff --git a/cmd/bd/init.go b/cmd/bd/init.go index 7f2e108de..55664ca23 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -28,9 +28,6 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" "github.com/steveyegge/beads/internal/storage/dolt" - beadsmysql "github.com/steveyegge/beads/internal/storage/mysql" - "github.com/steveyegge/beads/internal/storage/pgdialect" - "github.com/steveyegge/beads/internal/storage/postgres" "github.com/steveyegge/beads/internal/storage/schema" beadssqlite "github.com/steveyegge/beads/internal/storage/sqlite" "github.com/steveyegge/beads/internal/templates/agents" @@ -49,7 +46,7 @@ var initCmd = &cobra.Command{ and its storage (a Dolt database by default). Optionally specify a custom issue prefix. Dolt is the default backend and the only one with version control (history, -branching, sync). Select an alternative with --backend=; +branching, sync). Select the embedded SQLite alternative with --backend=sqlite; see docs/architecture/storage-backends.md for the trade-offs and setup. Use --database to specify an existing server database name, overriding the @@ -105,11 +102,6 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): serverUser, _ := cmd.Flags().GetString("server-user") database, _ := cmd.Flags().GetString("database") destroyToken, _ := cmd.Flags().GetString("destroy-token") - // Postgres backend flags (backend="postgres") - pgURL, _ := cmd.Flags().GetString("pg-url") - pgSchema, _ := cmd.Flags().GetString("pg-schema") - mysqlURL, _ := cmd.Flags().GetString("mysql-url") - mysqlDatabase, _ := cmd.Flags().GetString("mysql-database") sqlitePath, _ := cmd.Flags().GetString("sqlite-path") // --force is a deprecated alias for --reinit-local. They share @@ -249,25 +241,27 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): externalConfig = &cfg } - // Backend selection: dolt (default) + the pluggable SQL-family backends - // (postgres, mysql, sqlite). SQLite was removed under the one-backend - // assumption (#3151) and re-added as a pluggable leaf backend. - isPostgres := backendFlag == configfile.BackendPostgres - isMySQL := backendFlag == configfile.BackendMySQL + // Backend selection: Dolt (default) and the embedded SQLite backend. isSQLite := backendFlag == configfile.BackendSQLite - if backendFlag != "" && backendFlag != configfile.BackendDolt && !isPostgres && !isMySQL && !isSQLite { - return fmt.Errorf("unknown backend %q: supported backends are \"dolt\" (default), \"postgres\", \"mysql\", and \"sqlite\"", backendFlag) + if !configfile.IsSupportedBackend(backendFlag) { + if backendFlag == configfile.BackendPostgres || backendFlag == configfile.BackendMySQL { + return fmt.Errorf("storage backend %q is no longer supported: %s; supported backends are \"dolt\" (default) and \"sqlite\"", backendFlag, configfile.RemovedBackendRationale) + } + return fmt.Errorf("unknown backend %q: supported backends are \"dolt\" (default) and \"sqlite\"", backendFlag) } - if isPostgres || isMySQL || isSQLite { - // A non-Dolt SQL backend is a plain local workspace: only a small set of - // init-local flags apply. Reject any other init-local flag by ALLOWLIST — a + for _, legacyFlag := range removedServerBackendInitFlags { + if cmd.Flags().Changed(legacyFlag.name) { + return fmt.Errorf("--%s belonged to the removed PostgreSQL/MySQL initialization paths: %s; use --backend=dolt or --backend=sqlite", legacyFlag.name, configfile.RemovedBackendRationale) + } + } + if isSQLite { + // SQLite is a plain local workspace: only a small set of init-local flags + // apply. Reject any other init-local flag by ALLOWLIST — a // denylist silently ignores Dolt-only flags added later, the exact footgun // this block exists to prevent. Inherited/global flags (--json, --dir, …) // are not init-local and are left untouched. - pgAllowedFlags := map[string]bool{ - "backend": true, - "pg-url": true, "pg-schema": true, - "mysql-url": true, "mysql-database": true, + sqliteAllowedFlags := map[string]bool{ + "backend": true, "sqlite-path": true, "prefix": true, "quiet": true, "skip-hooks": true, "skip-agents": true, @@ -282,13 +276,13 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): if cmd.InheritedFlags().Lookup(f.Name) != nil { return } - if !pgAllowedFlags[f.Name] { + if !sqliteAllowedFlags[f.Name] { rejected = append(rejected, "--"+f.Name) } }) if len(rejected) > 0 { sort.Strings(rejected) - return fmt.Errorf("bd init --backend=%s does not support %s (a non-Dolt SQL backend is a plain local workspace: no Dolt server, sync, remote, or wizard)", backendFlag, strings.Join(rejected, ", ")) + return fmt.Errorf("bd init --backend=%s does not support %s (SQLite is a plain local workspace: no Dolt server, sync, remote, or wizard)", backendFlag, strings.Join(rejected, ", ")) } } @@ -321,8 +315,7 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): return fmt.Errorf("--team requires interactive prompts and cannot be used with --non-interactive") } - // Non-Dolt backends (postgres/mysql/sqlite) were dispatched and returned - // earlier; this is the Dolt init path. + // Dolt remains the default; the SQLite arm returns before Dolt provisioning. backend := configfile.BackendDolt // Also treat BEADS_DOLT_SERVER_MODE=1 env var as --server. @@ -815,31 +808,6 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): ctx := rootCtx - // Postgres proof-wedge: finalize a plain local workspace backed by Postgres. - // Everything below provisions Dolt (embedded/server DB dir, migrations, remote - // gate, sync, remotes) — none of which Postgres has an analog for — so the PG - // path provisions the schema + seeds + metadata.json here and returns. The - // shared setup above (.beads/, .gitignore, git init) already ran. - if isPostgres { - return runInitPostgres(ctx, initPostgresInput{ - beadsDir: beadsDir, - prefix: prefix, - pgURL: pgURL, - pgSchema: pgSchema, - quiet: quiet, - jsonOutput: jsonOutput, - }) - } - if isMySQL { - return runInitMySQL(ctx, initMySQLInput{ - beadsDir: beadsDir, - prefix: prefix, - mysqlURL: mysqlURL, - mysqlDatabase: mysqlDatabase, - quiet: quiet, - jsonOutput: jsonOutput, - }) - } if isSQLite { return runInitSQLite(ctx, initSQLiteInput{ beadsDir: beadsDir, @@ -1829,210 +1797,6 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): }, } -// initPostgresInput carries the resolved inputs the Postgres init arm needs from -// the shared prologue of the init command. -type initPostgresInput struct { - beadsDir string - prefix string - pgURL string - pgSchema string - quiet bool - jsonOutput bool -} - -// runInitPostgres finalizes a Postgres-backed workspace. It is the Postgres analog -// of the Dolt store-open + config-seed + metadata block (init.go), minus every -// version-control / server / sync step Postgres has no analog for: it provisions the -// schema (DDL + config-default seeds, idempotent), seeds the issue prefix and project -// identity, and writes metadata.json with the password-stripped DSN. The password is -// supplied at open time via BEADS_PG_PASSWORD and never lands on disk. -func runInitPostgres(ctx context.Context, in initPostgresInput) error { - dsn := in.pgURL - if dsn == "" { - dsn = os.Getenv("BEADS_POSTGRES_URL") - } - if dsn == "" { - return fmt.Errorf("bd init --backend=postgres requires --pg-url= (or BEADS_POSTGRES_URL); a password may be included for init but is never persisted — set BEADS_PG_PASSWORD for later commands") - } - schema := in.pgSchema - if schema == "" { - return fmt.Errorf("bd init --backend=postgres requires --pg-schema= (the per-workspace schema for search_path isolation)") - } - - // Redact BEFORE provisioning: a connection string whose password we cannot safely - // strip fails the whole init before any schema is created, and the password is - // guaranteed never to reach metadata.json (RedactPassword verifies with pgx that - // no password survives, failing closed). - persistDSN, err := pgdialect.RedactPassword(dsn) - if err != nil { - return fmt.Errorf("cannot safely persist the Postgres connection: %w", err) - } - - store, err := postgres.Provision(ctx, dsn, schema) - if err != nil { - return fmt.Errorf("failed to initialize Postgres workspace: %w", err) - } - defer func() { _ = store.Close() }() - - // Load any existing metadata.json up front so we can adopt its project identity. - cfg, _ := configfile.Load(in.beadsDir) - if cfg == nil { - cfg = configfile.DefaultConfig() - } - - // Seed the issue prefix, only if unset — a shared schema may already carry one. - // Matches the Dolt init path (dots normalized to underscores for valid IDs). - if in.prefix != "" { - if existing, _ := store.GetConfig(ctx, "issue_prefix"); existing == "" { - issuePrefix := strings.ReplaceAll(in.prefix, ".", "_") - if err := store.SetConfig(ctx, "issue_prefix", issuePrefix); err != nil { - return fmt.Errorf("failed to set issue prefix: %w", err) - } - } - } - - // Project identity: the DB is authoritative when it already carries one; otherwise - // adopt the metadata.json id (a re-init or committed clone) or mint a fresh UUID. - // Write it to BOTH the DB and metadata.json so the two can never diverge (GH#2372) — - // the Dolt path keeps them in lockstep the same way. - projectID, _ := store.GetMetadata(ctx, "_project_id") - if projectID == "" { - if cfg.ProjectID != "" { - projectID = cfg.ProjectID - } else { - projectID = configfile.GenerateProjectID() - } - if err := store.SetMetadata(ctx, "_project_id", projectID); err != nil { - return fmt.Errorf("failed to write project ID: %w", err) - } - } - - // Persist metadata.json: backend + password-free DSN + schema + the DB's identity. - cfg.Backend = configfile.BackendPostgres - cfg.PostgresDSN = persistDSN - cfg.PostgresSchema = schema - cfg.ProjectID = projectID - if err := cfg.Save(in.beadsDir); err != nil { - return fmt.Errorf("failed to write metadata.json: %w", err) - } - - switch { - case in.jsonOutput: - out := map[string]any{ - "status": "initialized", - "backend": configfile.BackendPostgres, - "schema": schema, - "prefix": in.prefix, - } - b, _ := json.MarshalIndent(out, "", " ") - fmt.Println(string(b)) - case !in.quiet: - fmt.Printf("%s bd initialized with the Postgres backend\n", ui.RenderPass("✓")) - fmt.Printf(" Schema: %s\n", schema) - if in.prefix != "" { - fmt.Printf(" Prefix: %s\n", in.prefix) - } - fmt.Println(" History: not tracked (Postgres backend has no version control; use Dolt for history)") - } - return nil -} - -// initMySQLInput carries the resolved inputs the MySQL init arm needs. -type initMySQLInput struct { - beadsDir string - prefix string - mysqlURL string - mysqlDatabase string - quiet bool - jsonOutput bool -} - -// runInitMySQL finalizes a MySQL-backed workspace: it creates the per-workspace -// database, applies the schema (config seeds on first provision), seeds the issue -// prefix and project identity, and writes metadata.json with the password-stripped -// DSN. The password is supplied at open time via BEADS_MYSQL_PASSWORD and never lands -// on disk. MySQL isolates by database where Postgres isolates by schema. -func runInitMySQL(ctx context.Context, in initMySQLInput) error { - dsn := in.mysqlURL - if dsn == "" { - dsn = os.Getenv("BEADS_MYSQL_URL") - } - if dsn == "" { - return fmt.Errorf("bd init --backend=mysql requires --mysql-url= (or BEADS_MYSQL_URL), e.g. user:pass@tcp(host:3306)/ ; a password may be included for init but is never persisted — set BEADS_MYSQL_PASSWORD for later commands") - } - database := in.mysqlDatabase - if database == "" { - return fmt.Errorf("bd init --backend=mysql requires --mysql-database= (the per-workspace database)") - } - - // Redact BEFORE provisioning so a DSN we cannot safely persist fails before any - // side effect, and the password is guaranteed never to reach metadata.json. - persistDSN, err := beadsmysql.RedactPassword(dsn) - if err != nil { - return fmt.Errorf("cannot safely persist the MySQL connection: %w", err) - } - - store, err := beadsmysql.Provision(ctx, dsn, database) - if err != nil { - return fmt.Errorf("failed to initialize MySQL workspace: %w", err) - } - defer func() { _ = store.Close() }() - - cfg, _ := configfile.Load(in.beadsDir) - if cfg == nil { - cfg = configfile.DefaultConfig() - } - - if in.prefix != "" { - if existing, _ := store.GetConfig(ctx, "issue_prefix"); existing == "" { - issuePrefix := strings.ReplaceAll(in.prefix, ".", "_") - if err := store.SetConfig(ctx, "issue_prefix", issuePrefix); err != nil { - return fmt.Errorf("failed to set issue prefix: %w", err) - } - } - } - - projectID, _ := store.GetMetadata(ctx, "_project_id") - if projectID == "" { - if cfg.ProjectID != "" { - projectID = cfg.ProjectID - } else { - projectID = configfile.GenerateProjectID() - } - if err := store.SetMetadata(ctx, "_project_id", projectID); err != nil { - return fmt.Errorf("failed to write project ID: %w", err) - } - } - - cfg.Backend = configfile.BackendMySQL - cfg.MySQLDSN = persistDSN - cfg.MySQLDatabase = database - cfg.ProjectID = projectID - if err := cfg.Save(in.beadsDir); err != nil { - return fmt.Errorf("failed to write metadata.json: %w", err) - } - - switch { - case in.jsonOutput: - out := map[string]any{ - "status": "initialized", - "backend": configfile.BackendMySQL, - "database": database, - "prefix": in.prefix, - } - b, _ := json.MarshalIndent(out, "", " ") - fmt.Println(string(b)) - case !in.quiet: - fmt.Printf("%s bd initialized with the MySQL backend\n", ui.RenderPass("✓")) - fmt.Printf(" Database: %s\n", database) - if in.prefix != "" { - fmt.Printf(" Prefix: %s\n", in.prefix) - } - fmt.Println(" History: not tracked (MySQL backend has no version control; use Dolt for history)") - } - return nil -} - // initSQLiteInput carries the resolved inputs the SQLite init arm needs. type initSQLiteInput struct { beadsDir string @@ -2140,13 +1904,17 @@ func init() { initCmd.Flags().Bool("non-interactive", false, "Skip all interactive prompts (auto-detected in CI or non-TTY environments)") initCmd.Flags().String("role", "", "Set beads role without prompting: \"maintainer\" or \"contributor\"") - // Backend selection: dolt (default) plus the SQL-family backends (postgres/mysql/sqlite). - initCmd.Flags().String("backend", "", "Storage backend: dolt (default), postgres, mysql, or sqlite. See docs/architecture/storage-backends.md.") - initCmd.Flags().String("pg-url", "", "Postgres connection URL (with --backend=postgres). A password may be included for init but is never persisted; set BEADS_PG_PASSWORD for later commands. Falls back to BEADS_POSTGRES_URL.") - initCmd.Flags().String("pg-schema", "", "Postgres schema for this workspace's tables (with --backend=postgres; provides search_path isolation)") - initCmd.Flags().String("mysql-url", "", "MySQL server DSN (with --backend=mysql), e.g. user:pass@tcp(host:3306)/ . A password may be included for init but is never persisted; set BEADS_MYSQL_PASSWORD for later commands. Falls back to BEADS_MYSQL_URL.") - initCmd.Flags().String("mysql-database", "", "MySQL database for this workspace (with --backend=mysql; MySQL's isolation unit)") + // Backend selection: Dolt (default) or embedded SQLite. + initCmd.Flags().String("backend", "", "Storage backend: dolt (default) or sqlite. See docs/architecture/storage-backends.md.") initCmd.Flags().String("sqlite-path", "", "SQLite database file (with --backend=sqlite; relative to the beads dir, default beads.db)") + // Keep the short-lived server-backend flags as hidden parser tombstones. A + // legacy invocation can then reach the backend rollback guidance instead of + // failing early with an opaque "unknown flag" error. Current backends reject + // these flags explicitly above; they are never consumed as connection data. + for _, legacyFlag := range removedServerBackendInitFlags { + initCmd.Flags().String(legacyFlag.name, "", legacyFlag.usage) + _ = initCmd.Flags().MarkHidden(legacyFlag.name) + } // Dolt server connection flags initCmd.Flags().Bool("server", false, "Use external dolt sql-server instead of embedded engine") @@ -2179,6 +1947,16 @@ func init() { rootCmd.AddCommand(initCmd) } +var removedServerBackendInitFlags = []struct { + name string + usage string +}{ + {name: "pg-url", usage: "Legacy PostgreSQL connection URL (removed backend compatibility only)"}, + {name: "pg-schema", usage: "Legacy PostgreSQL schema (removed backend compatibility only)"}, + {name: "mysql-url", usage: "Legacy MySQL connection URL (removed backend compatibility only)"}, + {name: "mysql-database", usage: "Legacy MySQL database (removed backend compatibility only)"}, +} + // migrateOldDatabases detects and migrates old database files to beads.db func migrateOldDatabases(targetPath string, quiet bool) error { targetDir := filepath.Dir(targetPath) @@ -2267,36 +2045,14 @@ func alreadyInitialized(format string, args ...any) error { // A returned error that matches errWorkspaceAlreadyInitialized means a database // already exists (the benign, idempotent-skip case); any other error is // operational and must not be treated as success. -// checkExistingSQLBackend returns an "already initialized" guard error when cfg -// selects a non-Dolt SQL backend (Postgres/MySQL/SQLite), each of which is marked -// by metadata.json alone with no local DB directory to inspect. It returns nil for -// the Dolt backend so the caller runs the directory/server checks instead. Split -// out of checkExistingBeadsDataAt so the added SQL-backend branches don't inflate -// that function's complexity. +// checkExistingSQLBackend returns an "already initialized" guard for SQLite and a +// fail-loud tombstone error for workspaces created with a removed backend. These +// backends are marked by metadata.json alone, with no Dolt directory to inspect. func checkExistingSQLBackend(cfg *configfile.Config) error { + if err := validateConfiguredBackend(cfg); err != nil { + return err + } switch cfg.GetBackend() { - case configfile.BackendPostgres: - return alreadyInitialized(` -%s This workspace is already initialized with the Postgres backend (schema %q). - -To use it: - Just run bd commands normally (e.g., %s) - -To re-initialize (Postgres provisioning is idempotent): - bd init --backend=postgres --reinit-local ... - -Aborting.`, ui.RenderWarn("⚠"), cfg.GetPostgresSchema(), ui.RenderAccent("bd list")) - case configfile.BackendMySQL: - return alreadyInitialized(` -%s This workspace is already initialized with the MySQL backend (database %q). - -To use it: - Just run bd commands normally (e.g., %s) - -To re-initialize (MySQL provisioning is idempotent): - bd init --backend=mysql --reinit-local ... - -Aborting.`, ui.RenderWarn("⚠"), cfg.GetMySQLDatabase(), ui.RenderAccent("bd list")) case configfile.BackendSQLite: return alreadyInitialized(` %s This workspace is already initialized with the SQLite backend (%q). @@ -2319,20 +2075,24 @@ func checkExistingBeadsDataAt(beadsDir string, prefix string) error { } // metadata.json is authoritative for the configured backend, so resolve it once - // and dispatch. A non-Dolt SQL backend (Postgres/MySQL/SQLite) is marked by - // metadata alone — there is no local DB directory to inspect — so a plain + // and dispatch. SQLite and removed-backend tombstones are marked by metadata + // alone — there is no local Dolt directory to inspect — so a plain // `bd init` (which defaults to Dolt) must not silently repoint a live SQL // workspace to a fresh embedded Dolt DB and orphan its issues. --reinit-local - // /--force bypass this (handled by the caller). A config that fails to load falls - // through to the redirect/database-file checks below. + // /--force bypass this (handled by the caller). Invalid metadata must fail closed: + // without an explicit reinitialization request, init may not overwrite the only + // marker for an external or otherwise nonlocal database. cfg, cfgErr := configfile.Load(beadsDir) - if cfgErr == nil && cfg != nil { + if cfgErr != nil { + return fmt.Errorf("failed to load %s: %w; refusing to reinitialize automatically (restore the metadata or use --reinit-local after safeguarding existing data)", configfile.ConfigPath(beadsDir), cfgErr) + } + if cfg != nil { if guardErr := checkExistingSQLBackend(cfg); guardErr != nil { return guardErr } } - if cfgErr == nil && cfg != nil && cfg.GetBackend() == configfile.BackendDolt { + if cfg != nil && cfg.GetBackend() == configfile.BackendDolt { if cfg.IsDoltProxiedServerMode() { proxiedRoot, rootErr := resolveProxiedServerRootPath(beadsDir) if rootErr != nil { diff --git a/cmd/bd/init_embedded_test.go b/cmd/bd/init_embedded_test.go index 409237153..bef8b52c2 100644 --- a/cmd/bd/init_embedded_test.go +++ b/cmd/bd/init_embedded_test.go @@ -1387,14 +1387,11 @@ func TestEmbeddedInit(t *testing.T) { t.Errorf("backend: got %q, want %q", cfg.Backend, configfile.BackendSQLite) } - // Postgres is recognized, not an unknown backend: it fails only because the - // required connection config is absent (bdEnv strips any ambient DSN). - pgOut := bdInitFail(t, bd, "--backend", "postgres") - if strings.Contains(pgOut, "unknown backend") { - t.Errorf("postgres should be recognized, got unknown-backend error: %s", pgOut) - } - if !strings.Contains(pgOut, "--pg-url") { - t.Errorf("postgres init should request --pg-url, got: %s", pgOut) + for _, backend := range []string{"postgres", "mysql"} { + out := bdInitFail(t, bd, "--backend", backend) + if !strings.Contains(out, "no longer supported") { + t.Errorf("%s should report the backend rollback: %s", backend, out) + } } // A genuinely unsupported backend is still rejected. diff --git a/cmd/bd/init_test.go b/cmd/bd/init_test.go index c152c9a13..2ddd728a7 100644 --- a/cmd/bd/init_test.go +++ b/cmd/bd/init_test.go @@ -2396,28 +2396,28 @@ func TestInitBackendFlag(t *testing.T) { } }) - // Postgres is a recognized backend now: it must not be rejected as unknown, - // and it fails only because the required connection config is absent. - t.Run("postgres_is_recognized_and_requires_config", func(t *testing.T) { - tmpDir := t.TempDir() - beadsDir := filepath.Join(tmpDir, ".beads") + for _, backend := range []string{"postgres", "mysql"} { + t.Run(backend+"_is_no_longer_supported", func(t *testing.T) { + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") - cmd := exec.Command(bd, "init", "--backend", "postgres", "--quiet") - cmd.Dir = tmpDir - cmd.Env = initBackendTestEnv(beadsDir) - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatal("Expected non-zero exit for --backend=postgres without connection config") - } + cmd := exec.Command(bd, "init", "--backend", backend, "--quiet") + cmd.Dir = tmpDir + cmd.Env = initBackendTestEnv(beadsDir) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("Expected non-zero exit for --backend=%s", backend) + } - outStr := string(out) - if strings.Contains(outStr, "unknown backend") { - t.Errorf("postgres must be a recognized backend, not reported unknown: %s", outStr) - } - if !strings.Contains(outStr, "--pg-url") { - t.Errorf("Expected missing --pg-url guidance for postgres, got: %s", outStr) - } - }) + outStr := string(out) + if !strings.Contains(outStr, "no longer supported") { + t.Errorf("Expected rollback guidance for %s, got: %s", backend, outStr) + } + if !strings.Contains(outStr, `"dolt"`) || !strings.Contains(outStr, `"sqlite"`) { + t.Errorf("Expected supported backend guidance for %s, got: %s", backend, outStr) + } + }) + } // A genuinely unsupported backend value is still rejected up front. t.Run("unknown_backend_errors", func(t *testing.T) { diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 3929f8e51..0e1bb6765 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -33,9 +33,6 @@ import ( "github.com/steveyegge/beads/internal/routing" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dolt" - mysqlstore "github.com/steveyegge/beads/internal/storage/mysql" - "github.com/steveyegge/beads/internal/storage/pgdialect" - pgstore "github.com/steveyegge/beads/internal/storage/postgres" "github.com/steveyegge/beads/internal/storage/schema" sqlitestore "github.com/steveyegge/beads/internal/storage/sqlite" "github.com/steveyegge/beads/internal/storage/uow" @@ -992,9 +989,13 @@ var rootCmd = &cobra.Command{ if dbPath == "" { if bd := beads.FindBeadsDir(); bd != "" { - if cfg, _ := configfile.Load(bd); cfg != nil && (cfg.IsDoltProxiedServerMode() || cfg.GetBackend() == configfile.BackendPostgres || cfg.GetBackend() == configfile.BackendMySQL || cfg.GetBackend() == configfile.BackendSQLite) { - // A non-Dolt SQL (or proxied-server) workspace has no local Dolt - // database file; the .beads dir with metadata.json IS the workspace. + cfg, cfgErr := configfile.Load(bd) + if cfgErr != nil || cfg != nil && (cfg.IsDoltProxiedServerMode() || cfg.GetBackend() == configfile.BackendSQLite || !configfile.IsSupportedBackend(cfg.Backend)) { + // SQLite, proxied-server, and removed-backend workspaces may have no + // local Dolt database file. Invalid or unknown metadata likewise must + // reach config validation instead of becoming a generic "no database" + // result. metadata.json identifies the workspace so store selection can + // route or reject it explicitly. dbPath = bd } } @@ -1076,6 +1077,18 @@ var rootCmd = &cobra.Command{ return HandleError("%v", err) } + // Resolve the backend before version tracking, migration, server startup, or + // any store construction. PostgreSQL/MySQL values are retained as metadata + // tombstones so an existing workspace fails closed instead of falling through + // to a new, empty Dolt database. + cfg, cfgErr := configfile.Load(beadsDir) + if cfgErr != nil { + return HandleError("failed to load beads config from %s: %v (refusing to fall back to the embedded store; fix or restore metadata.json and retry)", beadsDir, cfgErr) + } + if backendErr := validateConfiguredBackend(cfg); backendErr != nil { + return HandleError("%v", backendErr) + } + // Set actor for audit trail actor = getActorWithGit() // Attach actor to the command span now that we have it. @@ -1134,10 +1147,6 @@ var rootCmd = &cobra.Command{ // result set and exit 0 (false-empty), which readers misinterpret as // "no work". Absent metadata.json (cfg == nil, cfgErr == nil) keeps // the fresh-repo embedded default below. - cfg, cfgErr := configfile.Load(beadsDir) - if cfgErr != nil { - return HandleError("failed to load beads config from %s: %v (refusing to fall back to the embedded store; fix or restore metadata.json and retry)", beadsDir, cfgErr) - } if cfg != nil { warnSharedServerEmbeddedMismatch(cfg) doltCfg.ProxiedServer = cfg.IsDoltProxiedServerMode() @@ -1179,9 +1188,7 @@ var rootCmd = &cobra.Command{ // command must not run (or fail) embedded opens even when the env var is set. // Dolt-only: the gateway credential command mints a Dolt server // username. IsSharedServerMode() forces ServerMode true with no backend - // guard, so without this check a shared-server config in config.yaml - // would run (and fail closed) the command for postgres/mysql/sqlite - // workspaces, which never present a server username. + // guard, so SQLite must not try to resolve a server username. if doltCfg.ServerMode && cfg.GetBackend() == configfile.BackendDolt { if _, credErr := dolt.ApplyGatewayCredential(rootCtx, cfg, doltCfg); credErr != nil { return HandleError("resolving dolt credential command: %v", credErr) @@ -1275,14 +1282,7 @@ var rootCmd = &cobra.Command{ // Removing them WILL cause unrecoverable data corruption and data loss. // Dolt manages these files itself; external interference is never safe. - if cfg != nil && cfg.GetBackend() == configfile.BackendPostgres { - // Postgres backend: open via the SQL-family bundle, bypassing the - // Dolt open path (the doltCfg above is built but unused here). - store, err = pgstore.NewFromConfig(rootCtx, beadsDir) - } else if cfg != nil && cfg.GetBackend() == configfile.BackendMySQL { - // MySQL backend: same SQL-family bundle, isolation by database. - store, err = mysqlstore.NewFromConfig(rootCtx, beadsDir) - } else if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { + if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { // SQLite backend: pure-Go file-based SQL-family bundle. store, err = sqlitestore.NewFromConfig(rootCtx, beadsDir) } else { @@ -1394,7 +1394,7 @@ var rootCmd = &cobra.Command{ uowProvider = nil } } else { - // Slice 5c: a NonCommitGraphBackend (Postgres, SQLite, ...) has no Dolt + // A NonCommitGraphBackend such as SQLite has no Dolt // commit graph, so skip the Dolt-only maintenance tail. UnwrapStore reaches // the concrete store past the HookFiringStore decorator; a store WITHOUT the // marker (every Dolt variant) leaves skipMaintenance false and runs the tail @@ -1572,7 +1572,7 @@ func checkBlockedEnvVars() error { for _, name := range blockedEnvVars { if os.Getenv(name) != "" { return fmt.Errorf("%s env var is not supported and has been removed to prevent data fragmentation.\n"+ - "The storage backend is set in .beads/metadata.json. To change it, use: bd migrate dolt", name) + "Unset %s; storage selection comes from .beads/metadata.json. To choose a different supported backend, follow 'bd help init-safety'; do not edit metadata.json by hand", name, name) } } return nil @@ -1747,15 +1747,9 @@ func envTruthyValue(v string) bool { return true } -// dsnFlags are the flags whose value is a connection string that may embed a -// password (bd init --backend=postgres/mysql). Their values get the full -// parser-backed DSN scrub; other args only get the unambiguous userinfo scrub. -var dsnFlags = map[string]bool{"--pg-url": true, "--mysql-url": true} - // secretFlagNames are long flag names whose entire value is an opaque credential -// that must never reach the bd.args telemetry span. Unlike dsnFlags (a DSN that is -// parsed so its structure survives), a secret flag's value is redacted wholesale. -// Only federation add-peer's --password currently qualifies. Its shorthand (-p) is +// that must never reach the bd.args telemetry span. The flag's value is redacted +// wholesale. Only federation add-peer's --password currently qualifies. Its shorthand (-p) is // resolved per command via secretFlagTokens so the same letter bound to // --priority/--prefix/--parallel on other commands is never redacted. var secretFlagNames = map[string]bool{"password": true} @@ -1784,40 +1778,38 @@ func secretFlagTokens(cmd *cobra.Command) map[string]bool { } // scrubArgsForTelemetry joins argv for the bd.args span attribute with any -// credential-bearing values redacted. --pg-url/--mysql-url may carry a password -// for init, and federation add-peer --password/-p carries a SQL password; -// RedactPassword protects metadata.json, but the raw argv would otherwise leak the -// secret into the telemetry root span and trace logs. -// -// A DSN flag's value is scrubbed through the parser-backed pgdialect logic so every -// password form pgx accepts is caught — URL userinfo, URL `?password=`/`?sslpassword=` -// query params, and libpq `password=`/`sslpassword=` keyword/value tokens — in both -// the `--pg-url=` and `--pg-url ` spellings. A secretFlags token's value is -// an opaque credential and is redacted wholesale across the `--password `, +// credential-bearing values redacted. A secretFlags token's value is redacted +// wholesale across the `--password `, // `--password=`, `-p `, `-p=`, and `-p` spellings pflag accepts. Every -// other arg still gets the narrow user:PASS@host userinfo scrub as defense in depth, -// without the broad keyword scan that would over-redact ordinary text. +// other arg gets a conservative DSN/userinfo scrub as defense in depth so a +// positional connection string cannot leak a password. func scrubArgsForTelemetry(argv []string, secretFlags map[string]bool) string { parts := make([]string, len(argv)) + redactNext := false for i, a := range argv { - if name, val, ok := strings.Cut(a, "="); ok { - if dsnFlags[name] { - // --pg-url= — scrub only the value, keep the flag name. - parts[i] = name + "=" + scrubDSNValue(val) - continue - } + if redactNext { + parts[i] = "xxxxx" + redactNext = false + continue + } + if name, value, ok := strings.Cut(a, "="); ok { if secretFlags[name] { // --password= / -p= — redact the whole value. parts[i] = name + "=xxxxx" continue } + if strings.HasPrefix(name, "-") { + scrubbed := scrubUserinfoPassword(scrubPotentialDSNPasswords(value)) + if scrubbed != value { + // Preserve an arbitrary flag name while parsing its equals-value as + // a possible DSN. Passing the whole token to url.Parse would treat + // the flag prefix as the URL scheme and miss query credentials. + parts[i] = name + "=" + scrubbed + continue + } + } } if i > 0 { - if dsnFlags[argv[i-1]] { - // following a bare --pg-url token. - parts[i] = scrubDSNValue(a) - continue - } if secretFlags[argv[i-1]] { // following a bare --password / -p token. parts[i] = "xxxxx" @@ -1829,7 +1821,14 @@ func scrubArgsForTelemetry(argv []string, secretFlags map[string]bool) string { parts[i] = short + "xxxxx" continue } - parts[i] = scrubUserinfoPassword(a) + if secretShorthandTakesSeparateValue(a, secretFlags) { + // -qp — a boolean shorthand cluster ending in the + // value-taking secret shorthand, with its value in the next token. + parts[i] = a + redactNext = true + continue + } + parts[i] = scrubUserinfoPassword(scrubPotentialDSNPasswords(a)) } return strings.Join(parts, " ") } @@ -1868,12 +1867,23 @@ func secretShorthandPrefix(a string, secretFlags map[string]bool) (string, bool) return "", false } -// scrubDSNValue redacts every password form from a connection-string value. The -// pgdialect pass covers pgx's URL (userinfo + query) and libpq keyword/value shapes; -// the trailing userinfo pass covers the MySQL user:PASS@tcp(...) DSN, which is not a -// scheme URL and so is invisible to the pgdialect extractor. -func scrubDSNValue(val string) string { - return scrubUserinfoPassword(pgdialect.ScrubDSNString(val, val)) +// secretShorthandTakesSeparateValue recognizes a boolean-shorthand cluster that +// ends in a registered secret shorthand with no attached value. For example, +// pflag parses "-qp secret" as -q followed by -p=secret. +func secretShorthandTakesSeparateValue(a string, secretFlags map[string]bool) bool { + if len(a) < 3 || a[0] != '-' || a[1] == '-' { + return false + } + for i := 1; i < len(a); i++ { + c := a[i] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { + return false + } + if secretFlags["-"+string(c)] { + return i == len(a)-1 + } + } + return false } // scrubUserinfoPassword redacts the password in a URL/DSN userinfo section diff --git a/cmd/bd/main_test.go b/cmd/bd/main_test.go index 6563ed78c..187cc130f 100644 --- a/cmd/bd/main_test.go +++ b/cmd/bd/main_test.go @@ -142,6 +142,12 @@ func TestBlockedEnvVars(t *testing.T) { if err != nil && !strings.Contains(err.Error(), tt.envVar) { t.Errorf("expected error to mention %s, got: %v", tt.envVar, err) } + if err != nil && !strings.Contains(err.Error(), "bd help init-safety") { + t.Errorf("expected error to point to safe reinitialization guidance, got: %v", err) + } + if err != nil && strings.Contains(err.Error(), "bd migrate dolt") { + t.Errorf("error must not recommend the removed backend-conversion command: %v", err) + } }) } diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index 6fdac6657..283c375a7 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -15,8 +15,6 @@ import ( "github.com/steveyegge/beads/internal/storage/dbproxy/util" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/storage/embeddeddolt" - beadsmysql "github.com/steveyegge/beads/internal/storage/mysql" - "github.com/steveyegge/beads/internal/storage/postgres" beadssqlite "github.com/steveyegge/beads/internal/storage/sqlite" ) @@ -110,11 +108,8 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltS // metadata.json (cfg == nil, err == nil) keeps the embedded default. return nil, fmt.Errorf("load %s: %w (refusing to fall back to the embedded store)", configfile.ConfigPath(beadsDir), err) } - if cfg != nil && cfg.GetBackend() == configfile.BackendPostgres { - return postgres.NewFromConfig(ctx, beadsDir) - } - if cfg != nil && cfg.GetBackend() == configfile.BackendMySQL { - return beadsmysql.NewFromConfig(ctx, beadsDir) + if err := validateConfiguredBackend(cfg); err != nil { + return nil, err } if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { return beadssqlite.NewFromConfig(ctx, beadsDir) @@ -199,14 +194,8 @@ func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.D // "database not found" the embedded open would produce. return nil, fmt.Errorf("load %s: %w (refusing to fall back to the embedded store)", configfile.ConfigPath(beadsDir), err) } - if cfg != nil && cfg.GetBackend() == configfile.BackendPostgres { - // Postgres has no read-only open mode in the wedge; a normal open is fine - // (reads don't mutate, and search_path is per-workspace). - return postgres.NewFromConfig(ctx, beadsDir) - } - if cfg != nil && cfg.GetBackend() == configfile.BackendMySQL { - // MySQL likewise has no separate read-only open in the wedge; reads don't mutate. - return beadsmysql.NewFromConfig(ctx, beadsDir) + if err := validateConfiguredBackend(cfg); err != nil { + return nil, err } if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { return beadssqlite.NewFromConfig(ctx, beadsDir) diff --git a/cmd/bd/store_factory_nocgo.go b/cmd/bd/store_factory_nocgo.go index 892eb0f0f..d66a380aa 100644 --- a/cmd/bd/store_factory_nocgo.go +++ b/cmd/bd/store_factory_nocgo.go @@ -10,8 +10,6 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/util" "github.com/steveyegge/beads/internal/storage/dolt" - beadsmysql "github.com/steveyegge/beads/internal/storage/mysql" - "github.com/steveyegge/beads/internal/storage/postgres" beadssqlite "github.com/steveyegge/beads/internal/storage/sqlite" ) @@ -57,13 +55,8 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltS // message below. return nil, fmt.Errorf("load %s: %w", configfile.ConfigPath(beadsDir), err) } - if cfg != nil && cfg.GetBackend() == configfile.BackendPostgres { - // Postgres needs no CGO (pure-Go pgx), so it works in the nocgo build too. - return postgres.NewFromConfig(ctx, beadsDir) - } - if cfg != nil && cfg.GetBackend() == configfile.BackendMySQL { - // MySQL (go-sql-driver) needs no CGO either. - return beadsmysql.NewFromConfig(ctx, beadsDir) + if err := validateConfiguredBackend(cfg); err != nil { + return nil, err } if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { // SQLite (modernc.org/sqlite) is pure-Go; no CGO. @@ -90,11 +83,8 @@ func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.D if err != nil { return nil, fmt.Errorf("load %s: %w", configfile.ConfigPath(beadsDir), err) } - if cfg != nil && cfg.GetBackend() == configfile.BackendPostgres { - return postgres.NewFromConfig(ctx, beadsDir) - } - if cfg != nil && cfg.GetBackend() == configfile.BackendMySQL { - return beadsmysql.NewFromConfig(ctx, beadsDir) + if err := validateConfiguredBackend(cfg); err != nil { + return nil, err } if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { return beadssqlite.NewFromConfig(ctx, beadsDir) diff --git a/cmd/bd/store_selection_removed_backend_test.go b/cmd/bd/store_selection_removed_backend_test.go new file mode 100644 index 000000000..66bd224f1 --- /dev/null +++ b/cmd/bd/store_selection_removed_backend_test.go @@ -0,0 +1,403 @@ +package main + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/configfile" +) + +func TestCommandStartupRejectsRemovedBackendsBeforeLocalWrites(t *testing.T) { + bd := buildBDForInitTests(t) + for _, backend := range []string{configfile.BackendPostgres, configfile.BackendMySQL} { + t.Run(backend, func(t *testing.T) { + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatalf("create beads dir: %v", err) + } + if err := (&configfile.Config{Backend: backend}).Save(beadsDir); err != nil { + t.Fatalf("save config: %v", err) + } + + cmd := exec.Command(bd, "list", "--json") + cmd.Dir = root + cmd.Env = append(removedBackendTestEnv(beadsDir), "BD_DISABLE_METRICS=1") + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("removed backend %s unexpectedly opened", backend) + } + message := strings.ToLower(string(out)) + for _, want := range []string{"no longer supported", "simple and resource-light", "was not opened", "export", "dolt", "sqlite"} { + if !strings.Contains(message, want) { + t.Errorf("startup error for %s missing %q: %s", backend, want, message) + } + } + + for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version"} { + path := filepath.Join(beadsDir, name) + if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { + t.Fatalf("removed backend %s created local state at %s (stat error: %v)", backend, path, statErr) + } + } + }) + } +} + +func TestCommandStartupRejectsUnknownBackendBeforeLocalWrites(t *testing.T) { + bd := buildBDForInitTests(t) + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create .beads: %v", err) + } + if err := (&configfile.Config{Backend: "mystery"}).Save(beadsDir); err != nil { + t.Fatalf("save unknown-backend metadata: %v", err) + } + + cmd := exec.Command(bd, "list", "--json") + cmd.Dir = root + cmd.Env = append(removedBackendTestEnv(beadsDir), "BD_DISABLE_METRICS=1") + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("unknown backend unexpectedly opened:\n%s", out) + } + message := string(out) + if strings.Contains(message, "no beads database found") { + t.Fatalf("unknown backend stopped at generic discovery instead of metadata guidance:\n%s", out) + } + for _, want := range []string{"not recognized", "opened or modified", "dolt", "sqlite"} { + if !strings.Contains(message, want) { + t.Errorf("unknown-backend error missing %q:\n%s", want, out) + } + } + for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version"} { + if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { + t.Fatalf("unknown backend created %s (stat error: %v)", name, statErr) + } + } +} + +func TestInitGuardRejectsCorruptMetadataBeforeWorkspaceWrites(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create .beads: %v", err) + } + metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) + metadata := []byte("{\n") + if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { + t.Fatalf("write corrupt metadata.json: %v", err) + } + + err := checkExistingBeadsDataAt(beadsDir, "guard") + if err == nil || !strings.Contains(err.Error(), "metadata.json") { + t.Fatalf("init guard error = %v, want corrupt metadata refusal", err) + } + after, readErr := os.ReadFile(metadataPath) + if readErr != nil { + t.Fatalf("read metadata after refusal: %v", readErr) + } + if string(after) != string(metadata) { + t.Fatalf("init guard rewrote corrupt metadata: before %q, after %q", metadata, after) + } + for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version"} { + if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { + t.Fatalf("init guard created %s (stat error: %v)", name, statErr) + } + } +} + +func TestDoctorRejectsUnknownBackendBeforeDoltChecks(t *testing.T) { + bd := buildBDForInitTests(t) + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create .beads: %v", err) + } + cfg := &configfile.Config{ + Backend: "mystery", + DoltMode: configfile.DoltModeServer, + DoltServerHost: "127.0.0.1", + DoltServerPort: 1, + } + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("save unknown-backend metadata: %v", err) + } + + cmd := exec.Command(bd, "doctor", "--server", "--json") + cmd.Dir = root + cmd.Env = append(removedBackendTestEnv(beadsDir), "BEADS_DOLT_AUTO_START=0", "BD_DISABLE_METRICS=1") + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("doctor unexpectedly ran Dolt checks for unknown backend:\n%s", out) + } + for _, want := range []string{"not recognized", "no storage database was opened or modified", "dolt", "sqlite"} { + if !strings.Contains(string(out), want) { + t.Errorf("doctor unknown-backend error missing %q:\n%s", want, out) + } + } + for _, name := range []string{"embeddeddolt", "dolt", "beads.db", "dolt-server.pid", "dolt-server.port"} { + if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { + t.Fatalf("doctor created %s for unknown backend (stat error: %v)", name, statErr) + } + } +} + +func TestLegacyInitFlagsReachRemovedBackendGuidance(t *testing.T) { + bd := buildBDForInitTests(t) + tests := []struct { + backend string + args []string + }{ + { + backend: configfile.BackendPostgres, + args: []string{ + "init", "--backend=postgres", + "--pg-url=postgres://beads@127.0.0.1:5432/beads", + "--pg-schema=legacy_workspace", + }, + }, + { + backend: configfile.BackendMySQL, + args: []string{ + "init", "--backend=mysql", + "--mysql-url=beads@tcp(127.0.0.1:3306)/", + "--mysql-database=legacy_workspace", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.backend, func(t *testing.T) { + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + cmd := exec.Command(bd, tt.args...) + cmd.Dir = root + cmd.Env = append(removedBackendTestEnv(beadsDir), "BD_DISABLE_METRICS=1") + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("legacy %s init unexpectedly succeeded: %s", tt.backend, out) + } + + message := string(out) + if strings.Contains(message, "unknown flag") { + t.Errorf("legacy %s init stopped at argument parsing instead of rollback guidance: %s", tt.backend, message) + } + for _, want := range []string{"no longer supported", "general-purpose server databases", "simple and resource-light", "dolt", "sqlite"} { + if !strings.Contains(message, want) { + t.Errorf("legacy %s init error missing %q: %s", tt.backend, want, message) + } + } + if _, statErr := os.Stat(beadsDir); !os.IsNotExist(statErr) { + t.Fatalf("rejected legacy %s init created workspace state (stat error: %v)", tt.backend, statErr) + } + }) + } +} + +func TestDoltAdministrativeCommandsRejectRemovedBackends(t *testing.T) { + bd := buildBDForInitTests(t) + commands := []struct { + name string + args []string + }{ + {name: "show", args: []string{"dolt", "show"}}, + {name: "status", args: []string{"dolt", "status"}}, + {name: "set", args: []string{"dolt", "set", "host", "127.0.0.1"}}, + {name: "test", args: []string{"dolt", "test"}}, + {name: "start", args: []string{"dolt", "start"}}, + {name: "stop", args: []string{"dolt", "stop"}}, + {name: "killall", args: []string{"dolt", "killall"}}, + {name: "clean-databases", args: []string{"dolt", "clean-databases", "--dry-run"}}, + } + + for _, backend := range []string{configfile.BackendPostgres, configfile.BackendMySQL} { + t.Run(backend, func(t *testing.T) { + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatalf("create beads dir: %v", err) + } + if err := (&configfile.Config{Backend: backend}).Save(beadsDir); err != nil { + t.Fatalf("save config: %v", err) + } + + for _, command := range commands { + t.Run(command.name, func(t *testing.T) { + cmd := exec.Command(bd, command.args...) + cmd.Dir = root + cmd.Env = append(removedBackendTestEnv(beadsDir), "BD_DISABLE_METRICS=1") + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("%q unexpectedly succeeded for removed backend %s: %s", strings.Join(command.args, " "), backend, out) + } + + message := strings.ToLower(string(out)) + for _, want := range []string{"no longer supported", "simple and resource-light", "was not opened", "export", "dolt", "sqlite"} { + if !strings.Contains(message, want) { + t.Errorf("%q error for %s missing %q: %s", strings.Join(command.args, " "), backend, want, message) + } + } + + for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version"} { + path := filepath.Join(beadsDir, name) + if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { + t.Fatalf("%q created local state for removed backend %s at %s (stat error: %v)", strings.Join(command.args, " "), backend, path, statErr) + } + } + }) + } + }) + } +} + +func TestDoltReadOnlyCommandsRejectUnknownBackend(t *testing.T) { + bd := buildBDForInitTests(t) + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create .beads: %v", err) + } + if err := (&configfile.Config{Backend: "mystery", DoltMode: configfile.DoltModeServer, DoltServerPort: 1}).Save(beadsDir); err != nil { + t.Fatalf("save unknown-backend metadata: %v", err) + } + + for _, args := range [][]string{{"dolt", "show"}, {"dolt", "status"}, {"dolt", "killall"}} { + cmd := exec.Command(bd, args...) + cmd.Dir = root + cmd.Env = append(removedBackendTestEnv(beadsDir), "BEADS_DOLT_AUTO_START=0", "BD_DISABLE_METRICS=1") + out, err := cmd.CombinedOutput() + if err == nil { + t.Errorf("bd %s unexpectedly treated unknown metadata as Dolt:\n%s", strings.Join(args, " "), out) + continue + } + if !strings.Contains(string(out), "not recognized") || !strings.Contains(string(out), "opened or modified") { + t.Errorf("bd %s error lacks fail-closed guidance:\n%s", strings.Join(args, " "), out) + } + } +} + +func TestDoltKillallRejectsCorruptMetadataBeforeServerCleanup(t *testing.T) { + bd := buildBDForInitTests(t) + root := t.TempDir() + beadsDir := filepath.Join(root, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create .beads: %v", err) + } + metadata := []byte("{\n") + metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) + if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { + t.Fatalf("write corrupt metadata: %v", err) + } + + cmd := exec.Command(bd, "dolt", "killall") + cmd.Dir = root + cmd.Env = append(removedBackendTestEnv(beadsDir), + "BEADS_DOLT_SHARED_SERVER=1", + "BEADS_DOLT_AUTO_START=0", + "BEADS_SHARED_SERVER_DIR="+filepath.Join(root, "shared-server"), + "BD_DISABLE_METRICS=1", + ) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("bd dolt killall unexpectedly ignored corrupt metadata:\n%s", out) + } + message := strings.ToLower(string(out)) + for _, want := range []string{"loading config", "parsing config"} { + if !strings.Contains(message, want) { + t.Errorf("bd dolt killall corrupt-metadata error missing %q:\n%s", want, out) + } + } + after, readErr := os.ReadFile(metadataPath) + if readErr != nil { + t.Fatalf("read metadata after refusal: %v", readErr) + } + if string(after) != string(metadata) { + t.Fatalf("bd dolt killall rewrote corrupt metadata: before %q, after %q", metadata, after) + } + for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version", "dolt-server.pid", "dolt-server.port"} { + if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { + t.Fatalf("bd dolt killall created %s before rejecting corrupt metadata (stat error: %v)", name, statErr) + } + } +} + +func removedBackendTestEnv(beadsDir string) []string { + var env []string + for _, entry := range os.Environ() { + if strings.HasPrefix(entry, "BEADS_") || strings.HasPrefix(entry, "BD_") { + continue + } + env = append(env, entry) + } + return append(env, "BEADS_DIR="+beadsDir) +} + +func TestStoreFactoriesRemovedBackendsFailLoud(t *testing.T) { + for _, backend := range []string{configfile.BackendPostgres, configfile.BackendMySQL} { + t.Run(backend, func(t *testing.T) { + beadsDir := t.TempDir() + cfg := &configfile.Config{Backend: backend} + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("save config: %v", err) + } + + factories := map[string]func(context.Context, string) (interface{ Close() error }, error){ + "read-write": func(ctx context.Context, dir string) (interface{ Close() error }, error) { + return newDoltStoreFromConfig(ctx, dir) + }, + "read-only": func(ctx context.Context, dir string) (interface{ Close() error }, error) { + return newReadOnlyStoreFromConfig(ctx, dir) + }, + } + + for name, open := range factories { + t.Run(name, func(t *testing.T) { + store, err := open(t.Context(), beadsDir) + if err == nil { + if store != nil { + _ = store.Close() + } + t.Fatalf("expected removed-backend error for %s", backend) + } + if !strings.Contains(err.Error(), "no longer supported") { + t.Fatalf("error should explain that %s is no longer supported: %v", backend, err) + } + guidance := strings.ToLower(err.Error()) + if !strings.Contains(guidance, "export") || !strings.Contains(guidance, "dolt") || !strings.Contains(guidance, "sqlite") { + t.Fatalf("error should provide safe migration guidance: %v", err) + } + for _, name := range []string{"embeddeddolt", "dolt", "beads.db"} { + path := filepath.Join(beadsDir, name) + if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { + t.Fatalf("removed backend %s created local storage at %s (stat error: %v)", backend, path, statErr) + } + } + }) + } + }) + } +} + +func TestRequireDoltBackend(t *testing.T) { + for _, cfg := range []*configfile.Config{nil, {}, {Backend: configfile.BackendDolt}} { + if err := requireDoltBackend(cfg); err != nil { + t.Fatalf("Dolt config %#v rejected: %v", cfg, err) + } + } + + if err := requireDoltBackend(&configfile.Config{Backend: configfile.BackendSQLite}); err == nil || !strings.Contains(err.Error(), "not using Dolt") { + t.Fatalf("SQLite Dolt-command guard error = %v, want non-Dolt explanation", err) + } + + for _, backend := range []string{configfile.BackendPostgres, configfile.BackendMySQL} { + err := requireDoltBackend(&configfile.Config{Backend: backend}) + if err == nil || !strings.Contains(err.Error(), "no longer supported") || !strings.Contains(err.Error(), "export") { + t.Fatalf("removed backend %q guard error = %v, want rollback and migration guidance", backend, err) + } + } +} diff --git a/cmd/bd/telemetry_redact.go b/cmd/bd/telemetry_redact.go new file mode 100644 index 000000000..0be0b2d1e --- /dev/null +++ b/cmd/bd/telemetry_redact.go @@ -0,0 +1,129 @@ +package main + +import ( + "net/url" + "regexp" + "sort" + "strings" +) + +// These expressions recognize password-bearing libpq keyword/value tokens. The +// scrubber is intentionally backend-neutral: command-line telemetry may receive a +// connection string even when bd does not implement that connection's backend. +const dsnWhitespace = `\t\n\f\r \v` + +var dsnPasswordValue = regexp.MustCompile(`(?i)(?:^|[` + dsnWhitespace + `])(?:password|sslpassword)[` + dsnWhitespace + `]*=[` + dsnWhitespace + `]*(?:'((?:[^'\\]|\\.)*)'|((?:[^` + dsnWhitespace + `\\]|\\.)+))`) + +// A libpq keyword/value connection string begins with a recognized connection +// parameter. Requiring the first key to be recognized avoids treating ordinary +// prose such as "transport=rail password=..." as a DSN while still covering valid +// minimal forms such as "password=..." and service-file forms. +var dsnLeadingKeyword = regexp.MustCompile(`(?i)^[` + dsnWhitespace + `]*(?:application_name|channel_binding|client_encoding|connect_timeout|dbname|fallback_application_name|gssencmode|gsslib|host|hostaddr|keepalives|keepalives_count|keepalives_idle|keepalives_interval|krbsrvname|load_balance_hosts|options|passfile|password|port|replication|requirepeer|service|servicefile|sslcert|sslcrl|sslcrldir|sslkey|sslmode|sslpassword|sslrootcert|sslsni|ssl_max_protocol_version|ssl_min_protocol_version|target_session_attrs|tcp_user_timeout|user)[` + dsnWhitespace + `]*=`) + +func scrubStructuredDSNPasswords(dsn, text string) string { + secrets := structuredDSNPasswords(dsn) + sort.Slice(secrets, func(i, j int) bool { return len(secrets[i]) > len(secrets[j]) }) + for _, secret := range secrets { + text = strings.ReplaceAll(text, secret, "xxxxx") + } + return text +} + +func scrubPotentialDSNPasswords(value string) string { + if strings.Contains(value, "://") { + return scrubStructuredDSNPasswords(value, value) + } + if dsnLeadingKeyword.MatchString(value) { + return scrubStructuredDSNPasswords(value, value) + } + return value +} + +func structuredDSNPasswords(dsn string) []string { + seen := make(map[string]bool) + var values []string + add := func(value string) { + if value == "" || seen[value] { + return + } + seen[value] = true + values = append(values, value) + } + + if parsed, err := url.Parse(dsn); err == nil && parsed.Scheme != "" { + if parsed.User != nil { + if password, ok := parsed.User.Password(); ok { + add(password) + add(rawURLUserinfoPassword(dsn)) + } + } + } + // Extract query credentials independently of url.Parse. A malformed escape in + // the path, userinfo, or fragment makes url.Parse reject the entire URL even + // when the password query pair itself is valid. Telemetry redaction must fail + // closed in that case. + addRawURLQueryPasswords(dsn, add) + + for _, match := range dsnPasswordValue.FindAllStringSubmatch(dsn, -1) { + add(match[1]) + add(match[2]) + } + return values +} + +func addRawURLQueryPasswords(dsn string, add func(string)) { + queryStart := strings.IndexByte(dsn, '?') + if queryStart < 0 { + return + } + query := dsn[queryStart+1:] + if fragment := strings.IndexByte(query, '#'); fragment >= 0 { + query = query[:fragment] + } + for _, pair := range strings.Split(query, "&") { + key, value, ok := strings.Cut(pair, "=") + if !ok { + continue + } + decodedKey, err := url.QueryUnescape(key) + if err != nil { + decodedKey = key + } + if !isDSNPasswordKey(key) && !isDSNPasswordKey(decodedKey) { + continue + } + add(value) + if decoded, err := url.QueryUnescape(value); err == nil { + add(decoded) + } + } +} + +func isDSNPasswordKey(key string) bool { + switch strings.ToLower(key) { + case "password", "sslpassword": + return true + default: + return false + } +} + +func rawURLUserinfoPassword(dsn string) string { + scheme := strings.Index(dsn, "://") + if scheme < 0 { + return "" + } + authority := dsn[scheme+len("://"):] + if end := strings.IndexAny(authority, "/?#"); end >= 0 { + authority = authority[:end] + } + at := strings.LastIndex(authority, "@") + if at < 0 { + return "" + } + _, password, ok := strings.Cut(authority[:at], ":") + if !ok { + return "" + } + return password +} diff --git a/cmd/bd/telemetry_scrub_test.go b/cmd/bd/telemetry_scrub_test.go index 740746cb0..dfce9f013 100644 --- a/cmd/bd/telemetry_scrub_test.go +++ b/cmd/bd/telemetry_scrub_test.go @@ -7,12 +7,10 @@ import ( "github.com/spf13/cobra" ) -// TestScrubArgsForTelemetry locks the invariant that no password embedded in a -// --pg-url/--mysql-url value survives into the bd.args telemetry span, across every -// DSN password form pgx/go-sql-driver accept and both the --flag=value and -// --flag value spellings. The pre-fix scrubber handled only URL userinfo, so -// query-param and libpq keyword/value passwords (the confirmed red-team vectors) -// leaked verbatim. +// TestScrubArgsForTelemetry locks the invariant that a positional connection +// string cannot leak an embedded password into the bd.args telemetry span. This +// defense is backend-neutral and remains useful even when bd does not implement +// the connection string's database backend. func TestScrubArgsForTelemetry(t *testing.T) { const secret = "s3cr3t-pw" cases := []struct { @@ -21,34 +19,44 @@ func TestScrubArgsForTelemetry(t *testing.T) { keep []string // non-secret structure that must survive redaction }{ { - name: "pg url userinfo =form", - argv: []string{"init", "--backend=postgres", "--pg-url=postgres://bts:" + secret + "@127.0.0.1:5432/db"}, - keep: []string{"--pg-url=postgres://bts:", "127.0.0.1:5432/db", "--backend=postgres"}, + name: "url userinfo", + argv: []string{"command", "postgres://bts:" + secret + "@127.0.0.1:5432/db"}, + keep: []string{"postgres://bts:", "127.0.0.1:5432/db"}, }, { - name: "pg url query param space form", - argv: []string{"init", "--pg-url", "postgres://bts@127.0.0.1:5432/db?password=" + secret}, + name: "url query param", + argv: []string{"command", "postgres://bts@127.0.0.1:5432/db?password=" + secret}, keep: []string{"127.0.0.1:5432/db", "password="}, }, { - name: "pg url sslpassword =form", - argv: []string{"init", "--pg-url=postgres://bts@h:5432/db?sslpassword=" + secret + "&sslmode=require"}, + name: "url sslpassword", + argv: []string{"command", "postgres://bts@h:5432/db?sslpassword=" + secret + "&sslmode=require"}, keep: []string{"sslmode=require"}, }, { - name: "pg libpq keyword/value =form single token", - argv: []string{"init", "--pg-url=host=127.0.0.1 user=bts password=" + secret + " dbname=db"}, + name: "keyword value connection string", + argv: []string{"command", "host=127.0.0.1 user=bts password=" + secret + " dbname=db"}, keep: []string{"host=127.0.0.1", "user=bts", "dbname=db"}, }, { - name: "mysql userinfo space form", - argv: []string{"init", "--mysql-url", "bts:" + secret + "@tcp(127.0.0.1:3306)/db"}, - keep: []string{"tcp(127.0.0.1:3306)/db", "bts:"}, + name: "keyword value connection string with whitespace around equals", + argv: []string{"command", "host = 127.0.0.1 password=" + secret}, + keep: []string{"host = 127.0.0.1", "password="}, }, { - name: "mysql userinfo =form", - argv: []string{"init", "--mysql-url=bts:" + secret + "@tcp(127.0.0.1:3306)/db"}, - keep: []string{"--mysql-url=bts:", "tcp(127.0.0.1:3306)/db"}, + name: "password-only keyword connection string", + argv: []string{"command", "password=" + secret}, + keep: []string{"password="}, + }, + { + name: "service keyword connection string", + argv: []string{"command", "service=production password=" + secret}, + keep: []string{"service=production", "password="}, + }, + { + name: "scheme-less userinfo", + argv: []string{"command", "bts:" + secret + "@tcp(127.0.0.1:3306)/db"}, + keep: []string{"tcp(127.0.0.1:3306)/db", "bts:"}, }, } for _, tc := range cases { @@ -78,11 +86,24 @@ func TestScrubArgsForTelemetryLeavesOrdinaryArgs(t *testing.T) { if out := scrubArgsForTelemetry(argv, nil); out != "create --title document the password= knob" { t.Fatalf("over-redacted ordinary args: got %q", out) } + argv = []string{"create", "--title", "transport=rail password=foo"} + if out := scrubArgsForTelemetry(argv, nil); out != "create --title transport=rail password=foo" { + t.Fatalf("over-redacted non-DSN key/value text: got %q", out) + } argv = []string{"weird", "postgres://u:leak@h:5432/db"} if out := scrubArgsForTelemetry(argv, nil); strings.Contains(out, "leak") { t.Fatalf("userinfo password not scrubbed as defense in depth: %q", out) } + + for _, arg := range []string{ + "postgres://u@h:5432/db?password=leak", + "host=h user=u password=leak dbname=db", + } { + if out := scrubArgsForTelemetry([]string{"weird", arg}, nil); strings.Contains(out, "leak") { + t.Fatalf("positional DSN password not scrubbed as defense in depth: %q", out) + } + } } // TestSecretFlagTokensResolvesByCommand proves -p is treated as secret only on the @@ -190,6 +211,14 @@ func TestScrubArgsForTelemetryRedactsShorthandCluster(t *testing.T) { name: "boolean shorthand -v then -p cluster", argv: []string{"federation", "add-peer", "partner", "h:3306/db", "-vp" + secret}, }, + { + name: "boolean shorthand -q then separate -p value", + argv: []string{"federation", "add-peer", "partner", "h:3306/db", "-qp", secret}, + }, + { + name: "boolean shorthand -v then separate -p value", + argv: []string{"federation", "add-peer", "partner", "h:3306/db", "-vp", secret}, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -220,3 +249,194 @@ func TestScrubArgsForTelemetryKeepsOverloadedShortFlag(t *testing.T) { t.Fatalf("over-redacted non-secret -p=prefix: got %q", out) } } + +// TestTelemetryRedactionURLInFlagEqualsValue covers URL values embedded in a +// --flag=value argv element. The flag prefix must not prevent URL query parsing. +func TestTelemetryRedactionURLInFlagEqualsValue(t *testing.T) { + const secret = "query-secret" + arg := "--remote=postgres://bts@db.example/beads?password=" + secret + out := scrubArgsForTelemetry([]string{"init", arg}, nil) + if strings.Contains(out, secret) { + t.Fatalf("PASSWORD LEAK: URL query password in --flag=value survived telemetry scrub: %q", out) + } +} + +// TestTelemetryRedactionMalformedURLQuery fails closed when a URL-shaped DSN has +// an invalid percent escape. net/url rejects the whole URL in that case, but the +// query password must still never reach the bd.args telemetry span. +func TestTelemetryRedactionMalformedURLQuery(t *testing.T) { + const secret = "query-secret" + arg := "postgres://bts@db.example/beads%zz?password=" + secret + "&sslmode=require" + out := scrubArgsForTelemetry([]string{"command", arg}, nil) + if strings.Contains(out, secret) { + t.Fatalf("PASSWORD LEAK: malformed URL query password survived telemetry scrub: %q", out) + } + if !strings.Contains(out, "password=xxxxx") || !strings.Contains(out, "sslmode=require") { + t.Fatalf("malformed URL query structure was not preserved after redaction: %q", out) + } +} + +// TestScrubArgsForTelemetryAdversarialDSNVectors ports the adversarial regression +// vectors from the deleted internal/storage/pgdialect redaction tests +// (redact_error_test.go, redact_matrix_test.go). Each vector was a real leak or a +// scrubber-bypass near miss in an earlier redactor, so they are pinned here against +// the argv scrubber — the telemetry sink is backend-neutral and keeps receiving +// these DSN shapes even though the backends that motivated them are gone. +// +// Historical failure modes locked in, per case: +// - percent-encoded userinfo/query passwords must vanish in BOTH the raw +// (SUPER%2ASECRET) and decoded (SUPER*SECRET) forms; +// - percent-encoded query KEYS (pass%77ord, sslpass%77ord, %70assword) decode to +// password keys and bypassed key matching done on the raw key only; +// - password=SUPER\ SECRET: pgx accepts the backslash-escaped space as part of a +// single value, but a \S+-shaped value regex stopped at the space and shipped +// with the tail " SECRET" left in cleartext; +// - a single-quoted libpq password containing spaces is one value, not several; +// - pgx treats \v as inter-token whitespace, but Go's regexp \s class does not +// include \v, so a (^|\s)-anchored password regex never matched after \v and the +// password leaked entirely; +// - when one collected secret is a prefix of another (userinfo "foo", query +// "fooACTUAL"), replacing the short secret first turned "fooACTUAL" into +// "xxxxxACTUAL" and the tail survived; replacement must be length-descending. +func TestScrubArgsForTelemetryAdversarialDSNVectors(t *testing.T) { + const secret = "SUPERSECRET" + cases := []struct { + name string + dsn string + leaks []string // every cleartext form that must NOT survive redaction + keep []string // non-secret structure that must survive redaction + }{ + { + name: "url userinfo percent-encoded password", + dsn: "postgres://u:SUPER%2ASECRET@h:5432/db", + leaks: []string{"SUPER%2ASECRET", "SUPER*SECRET"}, + keep: []string{"postgres://u:", "h:5432/db"}, + }, + { + name: "url userinfo percent-encoded special chars", + dsn: "postgres://bts:p%40ss%3Aw%2Frd-" + secret + "@127.0.0.1:5432/db", + leaks: []string{"p%40ss%3Aw%2Frd-" + secret, "p@ss:w/rd-" + secret}, + keep: []string{"postgres://bts:", "127.0.0.1:5432/db"}, + }, + { + name: "url query param percent-encoded value", + dsn: "postgres://u@h:5432/db?password=SUPER%2ASECRET", + leaks: []string{"SUPER%2ASECRET", "SUPER*SECRET"}, + keep: []string{"password=", "h:5432/db"}, + }, + { + name: "url query param percent-encoded key (pass%77ord)", + dsn: "postgres://u@h:5432/db?pass%77ord=" + secret, + leaks: []string{secret}, + keep: []string{"pass%77ord=", "h:5432/db"}, + }, + { + name: "url query sslpassword percent-encoded key (sslpass%77ord)", + dsn: "postgres://u@h:5432/db?sslpass%77ord=" + secret + "&sslmode=require", + leaks: []string{secret}, + keep: []string{"sslpass%77ord=", "sslmode=require"}, + }, + { + name: "url query param fully percent-encoded key (%70assword)", + dsn: "postgres://u@h:5432/db?%70assword=" + secret, + leaks: []string{secret}, + keep: []string{"%70assword=", "h:5432/db"}, + }, + { + name: "url query password with sslmode preserved", + dsn: "postgres://bts@127.0.0.1:5432/db?password=" + secret + "&sslmode=disable", + leaks: []string{secret}, + keep: []string{"sslmode=disable", "127.0.0.1:5432/db"}, + }, + { + name: "url userinfo password with sslmode and application_name preserved", + dsn: "postgres://bts:" + secret + "@127.0.0.1:5432/db?sslmode=require&application_name=beads", + leaks: []string{secret}, + keep: []string{"sslmode=require", "application_name=beads", "127.0.0.1:5432/db"}, + }, + { + name: "url userinfo password but no user", + dsn: "postgres://:" + secret + "@h:5432/db", + leaks: []string{secret}, + keep: []string{"h:5432/db"}, + }, + { + name: "keyword value password with backslash-escaped whitespace", + dsn: `host=h user=u password=SUPER\ SECRET dbname=db`, + // "SECRET" pins the historical shipped bug: the tail after the escaped + // space must not survive in cleartext. + leaks: []string{"SECRET", `SUPER\`}, + keep: []string{"host=h", "user=u", "dbname=db"}, + }, + { + name: "keyword value single-quoted password containing spaces", + dsn: "host=h user=u password='se cret " + secret + "' dbname=db", + leaks: []string{"se cret", secret}, + keep: []string{"host=h", "user=u", "dbname=db"}, + }, + { + name: "keyword value single-quoted password", + dsn: "host=h user=u password='" + secret + "' dbname=db", + leaks: []string{secret}, + keep: []string{"host=h", "user=u", "dbname=db"}, + }, + { + name: "keyword value vertical-tab separator", + dsn: "host=h user=u\vpassword=" + secret + " dbname=db", + leaks: []string{secret}, + keep: []string{"host=h", "dbname=db"}, + }, + { + name: "keyword value sslpassword with sslmode preserved", + dsn: "host=h user=u sslpassword=" + secret + " sslmode=require dbname=db", + leaks: []string{secret}, + keep: []string{"host=h", "sslmode=require", "dbname=db"}, + }, + { + name: "keyword value password but no user", + dsn: "host=h port=5432 password=" + secret + " dbname=db", + leaks: []string{secret}, + keep: []string{"host=h", "port=5432", "dbname=db"}, + }, + { + name: "overlapping secrets where one is a prefix of another", + dsn: "postgres://u:foo@h:5432/db?password=fooACTUAL", + leaks: []string{"ACTUAL", ":foo@"}, + keep: []string{"postgres://u:", "h:5432/db"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out := scrubArgsForTelemetry([]string{"command", tc.dsn}, nil) + for _, leak := range tc.leaks { + if strings.Contains(out, leak) { + t.Errorf("PASSWORD LEAK: %q survived telemetry scrub of %q: %q", leak, tc.dsn, out) + } + } + if !strings.Contains(out, "xxxxx") { + t.Errorf("expected redaction marker xxxxx in %q", out) + } + for _, k := range tc.keep { + if !strings.Contains(out, k) { + t.Errorf("expected %q to survive redaction in %q", k, out) + } + } + }) + } +} + +// TestScrubArgsForTelemetryPasswordlessDSNUnchanged ports the deleted matrix +// round-trip invariant: a DSN carrying no password must pass through the scrubber +// byte-for-byte unchanged, in both URL and libpq keyword/value form. Over-redaction +// here would destroy the non-secret operational signal the span exists to carry. +func TestScrubArgsForTelemetryPasswordlessDSNUnchanged(t *testing.T) { + for _, dsn := range []string{ + "postgres://bts@127.0.0.1:5432/db?application_name=beads&sslmode=require", + "host=127.0.0.1 port=5432 user=bts dbname=db sslmode=require", + } { + want := "command " + dsn + if out := scrubArgsForTelemetry([]string{"command", dsn}, nil); out != want { + t.Errorf("passwordless DSN altered by scrubber:\n in = %q\n out = %q", want, out) + } + } +} diff --git a/cmd/bd/version_tracking.go b/cmd/bd/version_tracking.go index acba00814..1ef580632 100644 --- a/cmd/bd/version_tracking.go +++ b/cmd/bd/version_tracking.go @@ -163,6 +163,10 @@ func autoMigrateOnVersionBump(beadsDir string) { if cfg == nil { cfg = configfile.DefaultConfig() } + if cfg.GetBackend() != configfile.BackendDolt { + debug.Logf("auto-migrate: skipping Dolt migration for backend %q", cfg.GetBackend()) + return + } if cfg.IsDoltProxiedServerMode() { debug.Logf("auto-migrate: skipping embedded migration, proxied-server handled after UOW provider init") diff --git a/cmd/bd/version_tracking_test.go b/cmd/bd/version_tracking_test.go index 01678adef..fa46d271b 100644 --- a/cmd/bd/version_tracking_test.go +++ b/cmd/bd/version_tracking_test.go @@ -434,5 +434,5 @@ func TestAutoMigrateOnVersionBump_NoDatabase(t *testing.T) { // TestAutoMigrateOnVersionBump_RefusesDowngrade, and TestAutoMigrateOnVersionBump_TracksMaxVersion // were removed because they depended on the SQLite storage backend (sqlite.New) for round-trip // persistence testing through autoMigrateOnVersionBump -> dolt.NewFromConfig. -// Since SQLite support has been removed, auto-migration is only exercised with Dolt backends. +// Automatic schema migration is exercised here through the Dolt backend. // Dolt-based migration testing is covered by TestInitDoltMetadata in init_test.go. diff --git a/default.nix b/default.nix index bb503ed7c..aef41c212 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,7 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-Ij4xnWQLre/uE48TGMJRSGrunLqfXQ8MVBA+NCKlhLE="; + vendorHash = "sha256-L6pzbuY6h0FXYBTk5rvIUTSI2tsYthoNb1N7h3hWvZs="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not diff --git a/docs/architecture/dolt.md b/docs/architecture/dolt.md index 5ad45c6b8..dbafb79ac 100644 --- a/docs/architecture/dolt.md +++ b/docs/architecture/dolt.md @@ -3,7 +3,10 @@ title: Dolt Backend for Beads description: "How beads uses Dolt for versioned issue storage: embedded vs server mode, remotes, sync, and backups" --- -Beads uses Dolt as its storage backend. Dolt provides a version-controlled SQL database with cell-level merge, native branching, and two deployment modes. +Dolt is Beads' default storage backend. It provides a version-controlled SQL +database with cell-level merge, native branching, and two deployment modes. +For the supported server-free SQLite alternative, see +[Storage Backends](/architecture/storage-backends). ## Why Dolt? @@ -43,9 +46,11 @@ gt dolt start # Start the Dolt server bd init --server # Initialize with server mode ``` -### Migrate from SQLite (Legacy) +### Migrate from Legacy SQLite Installations -If upgrading from an older version that used SQLite: +Current releases support SQLite through the storage interface. This section is +only for upgrades from older, pre-Dolt SQLite schemas that are not compatible +with the current SQLite implementation. > **Note:** The `bd migrate --to-dolt` command was removed in v0.58.0. > For pre-0.50 installations with JSONL data, use the migration script: @@ -442,6 +447,7 @@ dolt: | Variable | Purpose | |----------|---------| | `BEADS_DOLT_PASSWORD` | Server mode password (highest priority) | +| `BEADS_DOLT_CREDENTIAL_COMMAND` | Gateway identity command (token becomes the connection username) | | `BEADS_CREDENTIALS_FILE` | Path to credentials file (overrides default location) | | `BEADS_DOLT_SERVER_MODE` | Enable server mode (set to "1") | | `BEADS_DOLT_SERVER_HOST` | Server host (default: 127.0.0.1) | @@ -453,6 +459,31 @@ dolt: | `DOLT_REMOTE_PASSWORD` | Clone/push/pull auth password | | `BD_DOLT_AUTO_COMMIT` | Override auto-commit setting | +### Gateway Identity Credentials + +Authenticating gateway deployments can set +`BEADS_DOLT_CREDENTIAL_COMMAND` to a command that mints a short-lived identity +token. Beads runs the command, reads a bare token or supported JSON token +envelope from stdout, and presents that token as the MySQL connection username. +The gateway verifies the identity and routes the connection to the appropriate +Dolt database. + +This is an identity credential, not a database password: + +- A server user already preset by the opener takes precedence. Otherwise the + command runs before the static `BEADS_DOLT_SERVER_USER` or metadata user. +- A configured command that fails aborts the connection. Beads does not fall + back to the static `root` user. +- The command is read from the environment only and is never persisted in + workspace metadata. +- Gateway mode assumes an externally managed server, so Beads does not + auto-start a local `dolt sql-server`. +- Tokens containing `:`, `@`, or `/` are rejected because the MySQL DSN grammar + cannot represent them safely in the username field. + +For ordinary Dolt server deployments, use `BEADS_DOLT_PASSWORD` or the +credentials file below instead. + ### Credentials File For multi-server setups, you can store passwords in an INI-style credentials file diff --git a/docs/cli-reference/doctor.md b/docs/cli-reference/doctor.md index a95098089..6289f5451 100644 --- a/docs/cli-reference/doctor.md +++ b/docs/cli-reference/doctor.md @@ -22,6 +22,13 @@ This command checks: - .beads/.gitignore up to date - Metadata.json version tracking (LastBdVersion field) +Storage Availability: + Full diagnostics, --perf, --deep, --server, --migration, and + --check=validate currently require Dolt server mode. Embedded Dolt and + SQLite support --check=artifacts, --check=conventions, and + --check=pollution. --check-health has a limited hook-health fallback. + Unsupported combinations return a notice without changing storage. + Performance Mode (--perf): Run performance diagnostics on your database: - Times key operations (bd ready, bd list, bd show, etc.) @@ -61,18 +68,12 @@ Server Mode (--server): - Schema compatible: Can query beads tables? - Connection pool: Pool health metrics -Migration Validation Mode (--migration): - Run Dolt migration validation checks with machine-parseable output. - Use --migration=pre before migration to verify readiness: - - JSONL file exists and is valid (parseable, no corruption) - - All JSONL issues are present in SQLite (or explains discrepancies) - - No blocking issues prevent migration - Use --migration=post after migration to verify completion: - - Dolt database exists and is healthy - - All issues from JSONL are present in Dolt - - No data was lost during migration - - Dolt database has no locks or uncommitted changes - Combine with --json for machine-parseable output for automation. +Legacy Dolt Migration Validation Mode (--migration): + Retained for older SQLite-to-Dolt migration workflows and available only in + Dolt server mode. It does not inspect or migrate a current SQLite workspace, + and it is not the migration path for removed PostgreSQL or MySQL backends. + Use the storage-backends migration guide for current backend moves. Combine + with --json for machine-parseable diagnostic output. Agent Mode (--agent): Output diagnostics designed for AI agent consumption. Instead of terse @@ -119,9 +120,9 @@ Examples: bd doctor --check=validate --fix # Auto-fix data-integrity issues bd doctor --deep # Full graph integrity validation bd doctor --server # Dolt server mode health checks - bd doctor --migration=pre # Validate readiness for Dolt migration - bd doctor --migration=post # Validate Dolt migration completed - bd doctor --migration=pre --json # Machine-parseable migration validation + bd doctor --migration=pre # Legacy Dolt-server migration diagnostic + bd doctor --migration=post # Legacy Dolt-server completion diagnostic + bd doctor --migration=pre --json # Machine-parseable legacy diagnostic ``` bd doctor [path] [flags] @@ -139,7 +140,7 @@ bd doctor [path] [flags] --fix Automatically fix issues where possible --fix-child-parent Remove child→parent dependencies (opt-in) -i, --interactive Confirm each fix individually - --migration string Run Dolt migration validation: 'pre' (before migration) or 'post' (after migration) + --migration string Run legacy Dolt-server migration diagnostics: 'pre' or 'post' --orchestrator Running in orchestrator multi-workspace mode (routes.jsonl is expected, higher duplicate tolerance) --orchestrator-duplicates-threshold int Duplicate tolerance threshold for orchestrator mode (wisps are ephemeral) (default 1000) -o, --output string Export diagnostics to JSON file diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index caaf03bb3..e83df4097 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -71,9 +71,11 @@ The wizard will: - Install git hooks (skip with `--skip-hooks`) Notes: -- Dolt is the default (and only) storage backend. Data is stored in `.beads/embeddeddolt/`. +- Dolt is the default storage backend. Embedded data is stored in `.beads/embeddeddolt/`. - By default, Dolt runs in **embedded mode** (in-process, no server needed). - For multi-writer setups, use `bd init --server` to connect to a `dolt sql-server` instead. +- For a small server-free workspace without Dolt history or sync, use + `bd init --backend=sqlite`. - To import issues from an older installation, run `bd init --from-jsonl`. ### Role configuration diff --git a/docs/multi-agent/federation.md b/docs/multi-agent/federation.md index 107abb369..a696ab603 100644 --- a/docs/multi-agent/federation.md +++ b/docs/multi-agent/federation.md @@ -19,7 +19,8 @@ data between independent teams or locations. Key benefits: ## Prerequisites -1. **Dolt backend**: Federation requires the Dolt storage backend (the only supported backend) +1. **Dolt backend**: Federation requires embedded or server-mode Dolt; SQLite + does not provide the commit graph or remotes federation needs ## Configuration diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index eab6620ad..e416a0431 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -5,18 +5,26 @@ description: Complete reference for bd configuration across config.yaml and data Complete configuration reference for beads. -Last reviewed: 2026-07-10 +Last reviewed: 2026-07-16 Freshness source: `cmd/bd/main.go`, `cmd/bd/config.go`, and `internal/configfile/`. beads has two complementary configuration systems: 1. **Tool-level configuration** (YAML, managed by [Viper](https://github.com/spf13/viper)) — startup flags and tool behavior, stored in `config.yaml` files. These are user preferences: output format, auto-commit behavior, CLI ergonomics. -2. **Project-level configuration** (managed by `bd config`) — integration credentials, status maps, and project-specific settings, stored in the Dolt database. Some keys are routed to `config.yaml` instead (see [YAML-only keys](#yaml-only-keys-startup-settings) below). +2. **Project-level configuration** (managed by `bd config`) — integration credentials, status maps, and project-specific settings, stored in the active storage database. Some keys are routed to `config.yaml` instead (see [YAML-only keys](#yaml-only-keys-startup-settings) below). -The split is deliberate: tool settings are user-specific; project config is team-shared and travels with the database when you run `bd dolt push`. That is also why secrets are refused in the database — see [Security](#security-where-secrets-live). +The split is deliberate: tool settings are user-specific, while project config +travels with the active database. In a Dolt workspace it is team-shared when +you run `bd dolt push`; SQLite has no built-in cross-machine sync. Secrets are +refused in either database — see [Security](#security-where-secrets-live). -Dolt is the only storage backend. Embedded mode (the default) stores data at `.beads/embeddeddolt/`; server mode (`bd init --server` or `BEADS_DOLT_SERVER_MODE=1`) uses `.beads/dolt/`. See [Dolt architecture](/architecture/dolt). +Supported storage paths are embedded Dolt (the default), Dolt server, and +SQLite. Embedded Dolt stores data at `.beads/embeddeddolt/`; managed repo-local +server mode (`bd init --server`) uses `.beads/dolt/`, while shared and external +servers manage their own data locations. SQLite uses `.beads/beads.db` by +default. See +[Storage Backends](/architecture/storage-backends). ## Configuration Locations @@ -38,7 +46,9 @@ For Viper-managed (YAML) keys, highest to lowest: 3. **`config.yaml`** files (in the order listed above) 4. **Built-in defaults** -Project-level keys written via `bd config set` (Jira, Linear, GitHub, status maps, etc.) live in the Dolt database. They are read at command time and have no env var override. +Project-level keys written via `bd config set` (Jira, Linear, GitHub, status +maps, etc.) live in the active storage database. They are read at command time +and have no env var override. When a config.yaml value or environment variable shadows a database key, `bd config list` prints an override warning, and `bd config show` reports the source of every effective key. @@ -72,7 +82,10 @@ bd config validate bd config unset jira.url ``` -`bd config set` automatically routes the write to the right location: keys in the YAML namespace (see below) are written to the project `config.yaml`; everything else is written to the Dolt database. `beads.role` is stored in git config. +`bd config set` automatically routes the write to the right location: keys in +the YAML namespace (see below) are written to the project `config.yaml`; +everything else is written to the active storage database. `beads.role` is +stored in git config. Unrecognized keys produce a warning with a did-you-mean suggestion; use the `custom.*` namespace for user-defined keys. @@ -159,17 +172,22 @@ bd config set export.auto true bd config set export.git-add true ``` -Use `bd dolt push` / `bd dolt pull` for sync and `bd backup` for restorable -database backups. +For Dolt workspaces, use `bd dolt push` / `bd dolt pull` for sync and +`bd backup` for restorable database backups. SQLite has no native remote or +Dolt backup path; `bd export` remains available for issue-level portability. -Routing note: `output.title-length` and `agents.file` are functionally tool-level settings, but `bd config set` writes them to the Dolt database. They are typically read from `config.yaml` when set there directly. +Routing note: `output.title-length` and `agents.file` are functionally +tool-level settings, but `bd config set` writes them to the active storage +database. They are typically read from `config.yaml` when set there directly. `bd config show` is the source of truth for what's currently effective on your machine, including provenance. ## Dolt History, Backup, and Push -Three post-write behaviors run after each successful write command, in this order: auto-commit, auto-backup, auto-push. +For Dolt workspaces, three post-write behaviors run after each successful write +command, in this order: auto-commit, auto-backup, auto-push. SQLite commits its +SQL transaction and skips this Dolt-specific maintenance tail. ### Auto-commit: SQL Commits vs Dolt Commits @@ -259,7 +277,8 @@ export BEADS_ACTOR="my-github-handle" ## Project-Level Settings (Database) -These are written to the Dolt database by `bd config set` and have no env var override. Common namespaces: +These are written to the active storage database by `bd config set` and have no +env var override. Common namespaces: | Namespace | Purpose | |---|---| @@ -353,7 +372,10 @@ bd config set max_hash_length "8" # Upper bound (default 8) ## Sync and Federation -Beads syncs exclusively through Dolt remotes (`bd dolt push` / `bd dolt pull`) with cell-level merge. Use `bd export` for issue portability and `bd backup` for restorable database backups. +Native sync and federation require Dolt. Dolt workspaces use remotes +(`bd dolt push` / `bd dolt pull`) with cell-level merge and `bd backup` for +restorable database backups. SQLite has no native remote; use `bd export` for +issue portability. Federation settings live in `config.yaml`: @@ -466,9 +488,14 @@ Integration secrets follow tracker-specific conventions: `LINEAR_API_KEY`, `GITH ## Security: Where Secrets Live -- Tokens and API keys are never stored in the Dolt database — database config is pushed to remotes, which would expose secrets and trip GitHub secret scanning. `bd config set` routes secret keys to the local `config.yaml` instead. +- Tokens and API keys are never stored in the active storage database. Dolt + database config can be pushed to remotes, which would expose secrets and trip + GitHub secret scanning; SQLite follows the same safe rule. `bd config set` + routes secret keys to the local `config.yaml` instead. - Writing a secret to a git-tracked `config.yaml` is refused unless you pass `--force-git-tracked`; environment variables are the safer default. -- `bd init` writes a `.beads/.gitignore` that keeps the database directories (`embeddeddolt/`, `dolt/`), runtime files, push state, and the federation credential key out of git. +- `bd init` writes a `.beads/.gitignore` that keeps the database storage + (`embeddeddolt/`, `dolt/`, and SQLite database files), runtime files, push + state, and the federation credential key out of git. ## Example `.beads/config.yaml` diff --git a/examples/library-usage/README.md b/examples/library-usage/README.md index 9eac33788..6f9774950 100644 --- a/examples/library-usage/README.md +++ b/examples/library-usage/README.md @@ -35,9 +35,9 @@ import ( func main() { ctx := context.Background() - // Find and open database - dbPath := beads.FindDatabasePath() - store, err := beads.Open(ctx, dbPath) + // Find the workspace and open its configured Dolt or SQLite implementation + beadsDir := beads.FindBeadsDir() + store, err := beads.OpenBestAvailable(ctx, beadsDir) if err != nil { log.Fatal(err) } @@ -145,8 +145,8 @@ type VCStorage struct { beads beads.Storage } -func NewVCStorage(ctx context.Context, dbPath string) (*VCStorage, error) { - store, err := beads.Open(ctx, dbPath) +func NewVCStorage(ctx context.Context, beadsDir string) (*VCStorage, error) { + store, err := beads.OpenBestAvailable(ctx, beadsDir) if err != nil { return nil, err } diff --git a/examples/library-usage/main.go b/examples/library-usage/main.go index 3596776d7..087d4bc03 100644 --- a/examples/library-usage/main.go +++ b/examples/library-usage/main.go @@ -16,16 +16,16 @@ import ( func main() { ctx := context.Background() - // Find the Beads database (looks for .beads/*.db in current/parent dirs) - dbPath := beads.FindDatabasePath() - if dbPath == "" { - log.Fatal("No Beads database found. Run 'bd init' first.") + // Find the Beads workspace so backend selection can follow metadata.json. + beadsDir := beads.FindBeadsDir() + if beadsDir == "" { + log.Fatal("No Beads workspace found. Run 'bd init' first.") } - fmt.Printf("Using database: %s\n\n", dbPath) + fmt.Printf("Using workspace: %s\n\n", beadsDir) - // Open the database - store, err := beads.Open(ctx, dbPath) + // Open the configured Dolt or SQLite implementation. + store, err := beads.OpenBestAvailable(ctx, beadsDir) if err != nil { log.Fatalf("Failed to open storage: %v", err) } diff --git a/examples/library-usage/main_test.go b/examples/library-usage/main_test.go index 92953c6ff..f6780eeee 100644 --- a/examples/library-usage/main_test.go +++ b/examples/library-usage/main_test.go @@ -18,11 +18,17 @@ func TestExampleCompiles(t *testing.T) { } defer os.RemoveAll(tmpDir) - dbPath := filepath.Join(tmpDir, "dolt") + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatalf("Failed to create .beads dir: %v", err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"sqlite","sqlite_path":"beads.db"}`), 0o600); err != nil { + t.Fatalf("Failed to create metadata.json: %v", err) + } // Open storage ctx := context.Background() - store, err := beads.Open(ctx, dbPath) + store, err := beads.OpenBestAvailable(ctx, beadsDir) if err != nil { t.Fatalf("Failed to open storage: %v", err) } diff --git a/go.mod b/go.mod index abfdb204a..7fc873288 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/dolthub/driver/v2 v2.2.0 github.com/dolthub/eventkit v0.0.0-20260611184414-99f5693e696a github.com/go-sql-driver/mysql v1.10.0 - github.com/jackc/pgx/v5 v5.10.0 github.com/olebedev/when v1.1.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 @@ -45,9 +44,6 @@ require ( github.com/cockroachdb/apd/v3 v3.2.3 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/moby/moby/api v1.54.1 // indirect github.com/moby/moby/client v0.4.0 // indirect diff --git a/go.sum b/go.sum index 29901bb04..4dbbe7535 100644 --- a/go.sum +++ b/go.sum @@ -724,7 +724,6 @@ github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bY github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= @@ -735,8 +734,6 @@ github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= @@ -747,14 +744,10 @@ github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9 github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= -github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= -github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= -github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= diff --git a/internal/beads/beads.go b/internal/beads/beads.go index baf010799..dc08adc22 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -421,13 +421,37 @@ func findLocalBeadsDir() string { } // findDatabaseInBeadsDir searches for a database within a .beads directory. -// Checks metadata.json for the Dolt database path. For server mode, no local -// directory is required. For embedded mode, checks both the embeddeddolt/ -// directory (where the embedded engine stores data) and the legacy dolt/ path. -// Returns empty string if no database is found. +// Checks metadata.json for the selected implementation's database path. For +// server mode and SQLite, no local path needs to exist yet: authoritative +// metadata is enough to route the caller without falling through to Dolt. +// Embedded Dolt checks both embeddeddolt/ and the legacy dolt/ path. Returns +// empty string if no database is configured or found. func findDatabaseInBeadsDir(beadsDir string, _ bool) string { // Check for metadata.json first (single source of truth) - if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil { + cfg, err := configfile.Load(beadsDir) + if err != nil { + // A present but unreadable/invalid metadata file is authoritative. Do + // not bypass it by discovering a leftover local Dolt directory; callers + // that need the detailed error should use OpenBestAvailable. + return "" + } + if cfg != nil { + if !configfile.IsSupportedBackend(cfg.Backend) { + // Unknown and removed backend markers are not Dolt aliases. Let the + // storage-selection layer report the metadata error without routing + // through a leftover local Dolt directory. + return "" + } + if cfg.GetBackend() == configfile.BackendSQLite { + dbPath := cfg.GetSQLitePath() + if dbPath == "" { + dbPath = "beads.db" + } + if !filepath.IsAbs(dbPath) { + dbPath = filepath.Join(beadsDir, dbPath) + } + return dbPath + } // For Dolt server mode, database is on the server - no local directory required if cfg.IsDoltServerMode() { return cfg.DatabasePath(beadsDir) @@ -491,6 +515,7 @@ func FindDatabasePath() string { // BEADS_DIR is set but no database found - this is OK for --no-db mode // Return empty string and let the caller handle it + return "" } // 2. Check BEADS_DB environment variable (deprecated but still supported) diff --git a/internal/beads/sqlite_database_path_test.go b/internal/beads/sqlite_database_path_test.go new file mode 100644 index 000000000..2939eee1a --- /dev/null +++ b/internal/beads/sqlite_database_path_test.go @@ -0,0 +1,85 @@ +package beads + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFindDatabasePathSQLiteMetadataOutsideCWD(t *testing.T) { + tests := []struct { + name string + metadata string + relativePath string + }{ + { + name: "default path", + metadata: `{"backend":"sqlite","sqlite_path":"beads.db"}`, + relativePath: "beads.db", + }, + { + name: "custom relative path", + metadata: `{"backend":"sqlite","sqlite_path":"data/custom.db"}`, + relativePath: filepath.Join("data", "custom.db"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + cwd := filepath.Join(root, "working-directory") + beadsDir := filepath.Join(root, "external-workspace", ".beads") + if err := os.MkdirAll(cwd, 0o750); err != nil { + t.Fatalf("create working directory: %v", err) + } + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatalf("create beads directory: %v", err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(tt.metadata), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + + t.Chdir(cwd) + t.Setenv("BEADS_DIR", beadsDir) + t.Setenv("BEADS_DB", "") + t.Setenv("BD_DB", "") + + want := filepath.Join(beadsDir, tt.relativePath) + if _, err := os.Stat(want); !os.IsNotExist(err) { + t.Fatalf("SQLite database unexpectedly exists before discovery: %v", err) + } + + if got := FindDatabasePath(); got != want { + t.Errorf("FindDatabasePath() = %q, want %q", got, want) + } + if _, err := os.Stat(want); !os.IsNotExist(err) { + t.Fatalf("FindDatabasePath created SQLite database path %q: %v", want, err) + } + }) + } +} + +func TestFindDatabasePathDoesNotBypassCorruptMetadata(t *testing.T) { + root := t.TempDir() + beadsDir := filepath.Join(root, "selected", ".beads") + leftoverDolt := filepath.Join(beadsDir, "embeddeddolt") + if err := os.MkdirAll(leftoverDolt, 0o750); err != nil { + t.Fatalf("create leftover Dolt directory: %v", err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte("{"), 0o600); err != nil { + t.Fatalf("write corrupt metadata.json: %v", err) + } + ambiguousCWD := filepath.Join(root, "ambient") + if err := os.MkdirAll(filepath.Join(ambiguousCWD, ".beads", "embeddeddolt"), 0o750); err != nil { + t.Fatalf("create ambient Dolt workspace: %v", err) + } + t.Chdir(ambiguousCWD) + + t.Setenv("BEADS_DIR", beadsDir) + t.Setenv("BEADS_DB", "") + t.Setenv("BD_DB", "") + + if got := FindDatabasePath(); got != "" { + t.Fatalf("FindDatabasePath() = %q with corrupt metadata, want empty path instead of leftover Dolt", got) + } +} diff --git a/internal/configfile/backend_messages.go b/internal/configfile/backend_messages.go new file mode 100644 index 000000000..db026a5a1 --- /dev/null +++ b/internal/configfile/backend_messages.go @@ -0,0 +1,38 @@ +package configfile + +import "fmt" + +// This file is the single source of truth for the user-facing removed-backend +// and unknown-backend messages. cmd/bd, the public beads package, and the Dolt +// store must all fail closed with the same text, so the wording is deliberately +// centralized here. The framing of these messages is pending a separate +// maintainer decision — reword them here, in one place, once that lands. + +// RemovedBackendRationale explains why direct PostgreSQL/MySQL support was +// removed. Shortened site-specific messages (e.g. bd init flag guidance) +// compose with this clause directly. +const RemovedBackendRationale = "direct support for general-purpose server databases was rolled back to keep Beads simple and resource-light" + +// BackendNotOpenedGuarantee is the fail-closed data-safety guarantee included +// in backend rejection errors: refusing a workspace never opens, creates, or +// modifies its storage. +const BackendNotOpenedGuarantee = "no storage database was opened or modified" + +// RemovedBackendDetail returns the shared body of every removed-backend error: +// the rationale, the untouched-data guarantee, and the migration path. Callers +// prepend a site-specific lead-in; RemovedBackendError carries the standard one. +func RemovedBackendDetail(backend string) string { + return fmt.Sprintf("%s; the configured %s database was not opened or modified; export it with a bd version that supports %s, then follow bd help init-safety to reinitialize with Dolt or SQLite and import the exported data", RemovedBackendRationale, backend, backend) +} + +// RemovedBackendError is the standard fail-closed error for metadata that +// selects a backend whose direct support was removed. +func RemovedBackendError(backend string) error { + return fmt.Errorf("storage backend %q is no longer supported: %s", backend, RemovedBackendDetail(backend)) +} + +// UnknownBackendError is the standard fail-closed error for metadata that +// names a backend this build does not recognize. +func UnknownBackendError(backend string) error { + return fmt.Errorf("storage backend %q in metadata.json is not recognized or supported; %s; supported backends are %q and %q; fix or restore metadata.json and retry", backend, BackendNotOpenedGuarantee, BackendDolt, BackendSQLite) +} diff --git a/internal/configfile/configfile.go b/internal/configfile/configfile.go index 712d2a4cf..d0c4351e8 100644 --- a/internal/configfile/configfile.go +++ b/internal/configfile/configfile.go @@ -17,7 +17,7 @@ const ConfigFileName = "metadata.json" type Config struct { Database string `json:"database"` - Backend string `json:"backend,omitempty"` // Storage backend: "dolt" (default), "postgres", "mysql", or "sqlite". Read via GetBackend(). + Backend string `json:"backend,omitempty"` // Storage backend: "dolt" (default) or "sqlite"; legacy "postgres"/"mysql" values are rejection tombstones. Read via GetBackend(). // Deletions configuration DeletionsRetentionDays int `json:"deletions_retention_days,omitempty"` // 0 means use default (3 days) @@ -36,15 +36,13 @@ type Config struct { DoltRemotesAPIPort int `json:"dolt_remotesapi_port,omitempty"` // Dolt remotesapi port for federation (default: 8080) // Note: Password should be set via BEADS_DOLT_PASSWORD env var for security - // Postgres backend (backend="postgres"). Password is NEVER persisted; it - // comes from BEADS_PG_PASSWORD or BEADS_POSTGRES_URL. - PostgresDSN string `json:"postgres_dsn,omitempty"` // e.g. postgres://user@host:5432/db (no password) - PostgresSchema string `json:"postgres_schema,omitempty"` // per-workspace schema (search_path) - - // MySQL backend (backend="mysql"). Password is NEVER persisted; it comes from - // BEADS_MYSQL_PASSWORD or BEADS_MYSQL_URL. - MySQLDSN string `json:"mysql_dsn,omitempty"` // e.g. user@tcp(host:3306)/ (no password) - MySQLDatabase string `json:"mysql_database,omitempty"` // per-workspace database (MySQL's isolation unit) + // Deprecated backend fields are retained only to round-trip metadata written by + // the short-lived PostgreSQL/MySQL implementations. They are not connection + // configuration for current builds; store selection rejects those backends. + PostgresDSN string `json:"postgres_dsn,omitempty"` + PostgresSchema string `json:"postgres_schema,omitempty"` + MySQLDSN string `json:"mysql_dsn,omitempty"` + MySQLDatabase string `json:"mysql_database,omitempty"` // SQLite backend (backend="sqlite"). File-based, embedded; no credentials. SQLitePath string `json:"sqlite_path,omitempty"` // database file, relative to the beads dir (default beads.db) @@ -231,9 +229,9 @@ type BackendCapabilities struct { SingleProcessOnly bool } -// CapabilitiesForBackend returns capabilities for a backend string. -// Dolt is the only supported backend. Returns SingleProcessOnly=true by default; -// use Config.GetCapabilities() to properly handle server mode. +// CapabilitiesForBackend returns capabilities for a backend string. Embedded Dolt +// and SQLite are single-process-only; use Config.GetCapabilities() to account for +// Dolt server and proxied-server modes. func CapabilitiesForBackend(_ string) BackendCapabilities { return BackendCapabilities{SingleProcessOnly: true} } @@ -249,10 +247,18 @@ func (c *Config) GetCapabilities() BackendCapabilities { return CapabilitiesForBackend(backend) } -// GetBackend returns the configured storage backend. Only the explicitly-allowlisted -// non-default backends ("postgres", "mysql") are honored; "", "dolt", and any legacy -// or unknown value resolve to dolt so the default path stays byte-identical and a -// typo fails safe to Dolt. +// IsSupportedBackend reports whether backend selects an implementation shipped by +// beads. The empty value is the legacy/default spelling of Dolt. +func IsSupportedBackend(backend string) bool { + return backend == "" || backend == BackendDolt || backend == BackendSQLite +} + +// GetBackend returns the configured storage backend. PostgreSQL and MySQL remain +// recognizable here so workspaces created by earlier releases can fail loudly at +// store selection instead of silently falling back to an empty Dolt database. +// Empty and explicit Dolt retain the established Dolt behavior. GetBackend keeps +// the historical Dolt fallback for unknown values, so storage-selection callers +// must check IsSupportedBackend(c.Backend) before opening or creating storage. func (c *Config) GetBackend() string { if c != nil { switch c.Backend { @@ -276,82 +282,6 @@ func (c *Config) GetSQLitePath() string { return c.SQLitePath } -// GetPostgresDSN returns the base Postgres connection string: BEADS_POSTGRES_URL (a -// full override that may carry a password) if set, else the persisted, -// password-free metadata postgres_dsn. It never merges a password — password -// resolution (BEADS_PG_PASSWORD_COMMAND, BEADS_PG_PASSWORD, the credentials file) -// and placement happen at open time in the postgres backend, which owns the pgx -// parser this low-level package must not import (see the note at RedactPassword). -func (c *Config) GetPostgresDSN() string { - if u := os.Getenv("BEADS_POSTGRES_URL"); u != "" { - return u - } - if c == nil { - return "" - } - return c.PostgresDSN -} - -// GetPostgresPasswordCommand returns the credential command that resolves the -// Postgres password: BEADS_PG_PASSWORD_COMMAND. Empty means no command — the static -// BEADS_PG_PASSWORD / credentials-file path applies. The command's stdout is a -// password (a bare token or a {token,expires_in} envelope); it is run at open time, -// out-ranking the static password so a rotating secret is never shadowed by a stale -// env value. It is deliberately read from the environment only, NOT metadata.json: -// a metadata-sourced command is arbitrary code run on open, so persisting it waits -// on a workspace-trust gate. -func (c *Config) GetPostgresPasswordCommand() string { - return os.Getenv("BEADS_PG_PASSWORD_COMMAND") -} - -// GetPostgresSchema returns the per-workspace Postgres schema (search_path). -func (c *Config) GetPostgresSchema() string { - if c == nil { - return "" - } - return c.PostgresSchema -} - -// GetMySQLDSN returns the base MySQL server DSN: BEADS_MYSQL_URL (a full override -// that may carry a password) if set, else the persisted, password-free metadata -// mysql_dsn. It never merges a password — password resolution -// (BEADS_MYSQL_PASSWORD_COMMAND, BEADS_MYSQL_PASSWORD, the credentials file) and -// placement happen at open time in the mysql backend, which owns the go-sql-driver -// parser this low-level package must not import (see the note at RedactPassword). -func (c *Config) GetMySQLDSN() string { - if u := os.Getenv("BEADS_MYSQL_URL"); u != "" { - return u - } - if c == nil { - return "" - } - return c.MySQLDSN -} - -// GetMySQLPasswordCommand returns the credential command that resolves the MySQL -// password: BEADS_MYSQL_PASSWORD_COMMAND. Empty means no command — the static -// BEADS_MYSQL_PASSWORD / credentials-file path applies. The command's stdout is a -// password (a bare token or a {token,expires_in} envelope); it is run at open time, -// out-ranking the static password so a rotating secret is never shadowed by a stale -// env value. It is deliberately read from the environment only, NOT metadata.json: -// a metadata-sourced command is arbitrary code run on open, so persisting it waits -// on a workspace-trust gate. -func (c *Config) GetMySQLPasswordCommand() string { - return os.Getenv("BEADS_MYSQL_PASSWORD_COMMAND") -} - -// GetMySQLDatabase returns the per-workspace MySQL database (isolation unit). -func (c *Config) GetMySQLDatabase() string { - if c == nil { - return "" - } - return c.MySQLDatabase -} - -// Password redaction for persistence lives in pgdialect.RedactPassword, not here: -// it must fail closed (verify with pgx that no password survives) and therefore -// depends on the pgx parser, which this low-level config package must not import. - // Dolt mode constants const ( DoltModeEmbedded = "embedded" diff --git a/internal/configfile/configfile_test.go b/internal/configfile/configfile_test.go index 9f9bbed8e..8bc1a725c 100644 --- a/internal/configfile/configfile_test.go +++ b/internal/configfile/configfile_test.go @@ -629,10 +629,10 @@ func TestProxiedServerClientInfo_ResolvedPaths(t *testing.T) { }) } -// TestGetBackendAllowlist verifies the allowlist semantics: the SQL backends -// (postgres, mysql, sqlite) are honored; every other value (empty, legacy, -// genuinely unknown) falls back to Dolt. This is the guard behind backend selection -// — a typo in metadata.json must fail safe to Dolt, never to an unintended backend. +// TestGetBackendAllowlist verifies the metadata-routing semantics: current backends +// and removed-backend tombstones remain recognizable, while empty or unknown values +// retain the historical Dolt fallback. Store selection rejects the tombstones before +// it can open an empty Dolt database. func TestGetBackendAllowlist(t *testing.T) { fallsBackToDolt := []struct { name string @@ -662,6 +662,29 @@ func TestGetBackendAllowlist(t *testing.T) { } } +func TestSupportedBackendAllowlist(t *testing.T) { + tests := []struct { + name string + backend string + supported bool + }{ + {name: "implicit dolt", backend: "", supported: true}, + {name: "dolt", backend: BackendDolt, supported: true}, + {name: "sqlite", backend: BackendSQLite, supported: true}, + {name: "postgres", backend: BackendPostgres, supported: false}, + {name: "mysql", backend: BackendMySQL, supported: false}, + {name: "unknown", backend: "mystery", supported: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsSupportedBackend(tt.backend); got != tt.supported { + t.Fatalf("IsSupportedBackend(%q) = %v, want %v", tt.backend, got, tt.supported) + } + }) + } +} + // TestDatabasePathAlwaysDolt tests that DatabasePath always returns the dolt path. func TestDatabasePathAlwaysDolt(t *testing.T) { beadsDir := "/home/user/project/.beads" diff --git a/internal/configfile/mysql_dsn_test.go b/internal/configfile/mysql_dsn_test.go deleted file mode 100644 index 7749a22f3..000000000 --- a/internal/configfile/mysql_dsn_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package configfile - -import "testing" - -// GetMySQLDSN returns the BASE DSN and never merges a password: BEADS_MYSQL_PASSWORD -// is resolved and placed at open time in the mysql backend (which owns the -// go-sql-driver parser), so the persisted, password-free metadata DSN comes back -// unchanged here. -func TestGetMySQLDSNReturnsBaseUnmerged(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD", "secret") - c := &Config{MySQLDSN: "bts@tcp(127.0.0.1:3306)/"} - if got, want := c.GetMySQLDSN(), "bts@tcp(127.0.0.1:3306)/"; got != want { - t.Fatalf("GetMySQLDSN() = %q, want %q (must not merge the password)", got, want) - } -} - -// BEADS_MYSQL_URL (full override) takes precedence over the metadata DSN. -func TestGetMySQLDSNEnvURLPrecedence(t *testing.T) { - t.Setenv("BEADS_MYSQL_URL", "root:pw@tcp(host:3306)/db") - c := &Config{MySQLDSN: "other@tcp(127.0.0.1:3306)/"} - if got, want := c.GetMySQLDSN(), "root:pw@tcp(host:3306)/db"; got != want { - t.Fatalf("GetMySQLDSN() = %q, want %q", got, want) - } -} - -// GetMySQLPasswordCommand reads BEADS_MYSQL_PASSWORD_COMMAND from the environment -// only (metadata is deliberately excluded until a workspace-trust gate exists). -func TestGetMySQLPasswordCommand(t *testing.T) { - c := &Config{} - if got := c.GetMySQLPasswordCommand(); got != "" { - t.Fatalf("GetMySQLPasswordCommand() = %q, want empty when unset", got) - } - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "get-credential") - if got, want := c.GetMySQLPasswordCommand(), "get-credential"; got != want { - t.Fatalf("GetMySQLPasswordCommand() = %q, want %q", got, want) - } -} diff --git a/internal/configfile/postgres_dsn_test.go b/internal/configfile/postgres_dsn_test.go deleted file mode 100644 index 0ffbc57d3..000000000 --- a/internal/configfile/postgres_dsn_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package configfile - -import "testing" - -// GetPostgresDSN returns the BASE DSN and never merges a password: BEADS_PG_PASSWORD -// is resolved and placed at open time in the postgres backend (which owns the pgx -// parser), so the persisted, password-free metadata DSN comes back unchanged here. -func TestGetPostgresDSNReturnsBaseUnmerged(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD", "secret") - c := &Config{PostgresDSN: "postgres://bts@127.0.0.1:5432/db"} - if got, want := c.GetPostgresDSN(), "postgres://bts@127.0.0.1:5432/db"; got != want { - t.Fatalf("GetPostgresDSN() = %q, want %q (must not merge the password)", got, want) - } -} - -// BEADS_POSTGRES_URL (full override) takes precedence over the metadata DSN. -func TestGetPostgresDSNEnvURLPrecedence(t *testing.T) { - t.Setenv("BEADS_POSTGRES_URL", "postgres://a:b@host:5432/db") - c := &Config{PostgresDSN: "postgres://other@127.0.0.1:5432/db"} - if got, want := c.GetPostgresDSN(), "postgres://a:b@host:5432/db"; got != want { - t.Fatalf("GetPostgresDSN() = %q, want %q", got, want) - } -} - -// GetPostgresPasswordCommand reads BEADS_PG_PASSWORD_COMMAND from the environment -// only (metadata is deliberately excluded until a workspace-trust gate exists). -func TestGetPostgresPasswordCommand(t *testing.T) { - c := &Config{} - if got := c.GetPostgresPasswordCommand(); got != "" { - t.Fatalf("GetPostgresPasswordCommand() = %q, want empty when unset", got) - } - t.Setenv("BEADS_PG_PASSWORD_COMMAND", "get-credential") - if got, want := c.GetPostgresPasswordCommand(), "get-credential"; got != want { - t.Fatalf("GetPostgresPasswordCommand() = %q, want %q", got, want) - } -} diff --git a/internal/storage/backend_marker.go b/internal/storage/backend_marker.go index 717b9614b..1fd674cdd 100644 --- a/internal/storage/backend_marker.go +++ b/internal/storage/backend_marker.go @@ -2,15 +2,14 @@ package storage // NonCommitGraphBackend is an affirmative *negative* marker: it is implemented // only by backends whose Commit/GetCurrentCommit/Push do NOT participate in a -// real version-control commit graph (e.g. Postgres, SQLite). cmd/bd's PostRun +// real version-control commit graph (currently SQLite). cmd/bd's PostRun // skips its Dolt-only maintenance tail (auto-commit, auto-export, auto-backup, // auto-push) when the active store implements this and returns true. // // The marker is negative and lives only on non-Dolt backends by design: a store // that does NOT implement it is assumed to be Dolt, so the default path runs the -// full maintenance tail unchanged and stays byte-identical. New SQL backends opt -// OUT of Dolt maintenance by implementing this; they never have to touch the -// Dolt constructors. +// full maintenance tail unchanged and stays byte-identical. A future non-Dolt +// implementation would opt out of Dolt maintenance by implementing this marker. type NonCommitGraphBackend interface { CommitGraphUnsupported() bool } diff --git a/internal/storage/conformance/audit_config_metadata_slots_repomtime.go b/internal/storage/conformance/audit_config_metadata_slots_repomtime.go index 5dc11eba2..116891e02 100644 --- a/internal/storage/conformance/audit_config_metadata_slots_repomtime.go +++ b/internal/storage/conformance/audit_config_metadata_slots_repomtime.go @@ -43,9 +43,8 @@ func auditFoldCount(all map[string]string, want string) int { // testAuditConfigKeyCaseSensitive pins the reference's config key-column collation. // The finding predicted Dolt's VARCHAR primary key would be case-insensitive, but // the embedded-Dolt oracle treats "MyKey" and "mykey" as DISTINCT keys: each holds -// its own value and GetAllConfig carries both rows. Postgres text and SQLite BINARY -// are case-sensitive too, so they match; the divergence suspect is real MySQL, whose -// server-default utf8mb4 collation is case-insensitive and would collapse the two. +// its own value and GetAllConfig carries both rows. SQLite's BINARY collation matches +// that behavior. func testAuditConfigKeyCaseSensitive(t *testing.T, f Factory) { s := f(t) c := ctx() diff --git a/internal/storage/conformance/audit_labels-comments-events.go b/internal/storage/conformance/audit_labels-comments-events.go index 8f2088cba..c842c8fab 100644 --- a/internal/storage/conformance/audit_labels-comments-events.go +++ b/internal/storage/conformance/audit_labels-comments-events.go @@ -245,7 +245,7 @@ func testAuditImportCommentSubSecond(t *testing.T, f Factory) { c := ctx() must(t, s.CreateIssue(c, withDefaults(&types.Issue{ID: "test-i", Title: "I"}), "a")) // Sub-second precision is intentionally backend-specific (Dolt datetime(0) rounds - // half-up; SQLite keeps the fraction; Postgres keeps microseconds), so we test the + // half-up; SQLite keeps the fraction), so we test the // portable contract every backend honors: a whole-second UTC timestamp round-trips // exactly. Higher fidelity on some backends is accepted, not asserted. ts := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) diff --git a/internal/storage/conformance/audit_molecule_wisp_batch_iter.go b/internal/storage/conformance/audit_molecule_wisp_batch_iter.go index 05c0ca51f..c56457bde 100644 --- a/internal/storage/conformance/audit_molecule_wisp_batch_iter.go +++ b/internal/storage/conformance/audit_molecule_wisp_batch_iter.go @@ -665,9 +665,8 @@ func testAuditGetNextChildID(t *testing.T, f Factory) { } } -// The self-heal scan (id LIKE parent.'%') is case-sensitive on every backend -// (Dolt/MySQL bin collation, Postgres "C", SQLite case_sensitive_like — see -// bd-oyvc2.10), so a different-cased sibling parent's children never advance +// The self-heal scan (id LIKE parent.'%') is case-sensitive on Dolt and SQLite +// (see bd-oyvc2.10), so a different-cased sibling parent's children never advance // this parent's counter. func testAuditGetNextChildIDCaseSensitive(t *testing.T, f Factory) { s := f(t) diff --git a/internal/storage/conformance/audit_search-counts-stats.go b/internal/storage/conformance/audit_search-counts-stats.go index 13acbde54..756769a1d 100644 --- a/internal/storage/conformance/audit_search-counts-stats.go +++ b/internal/storage/conformance/audit_search-counts-stats.go @@ -182,7 +182,7 @@ func testAuditReadyWorkDepCreatedAtParity(t *testing.T, f Factory) { } // countByColumnInTx emits COALESCE(priority, ”) GROUP BY priority; priority is -// integer NOT NULL. MySQL/Dolt coerce the int to a string key, then +// integer NOT NULL. Both maintained implementations return a string key, then // countGroupForTablesInTx prepends 'P'. Reference keys: P0/P1/P2. func testAuditCountByPriority(t *testing.T, f Factory) { s := f(t) @@ -366,7 +366,7 @@ func testAuditSearchIdenticalTimestampIDOrder(t *testing.T, f Factory) { } // SortBy="closed" emits ORDER BY closed_at DESC, id ASC. closed_at is nullable; -// MySQL/Dolt place NULLs LAST on DESC. The open (NULL-closed) rows follow the +// the storage contract places NULLs last on DESC. The open (NULL-closed) rows follow the // closed ones, ordered by id ASC. func testAuditSearchSortByClosedNullsLast(t *testing.T, f Factory) { s := f(t) @@ -409,9 +409,9 @@ func testAuditSearchTextIDBranchExternalRef(t *testing.T, f Factory) { } } -// LIKE over raw-cased operands is case-sensitive on every backend: Dolt/MySQL use -// the utf8mb4_0900_bin table collation, Postgres uses collation "C", and SQLite -// sets PRAGMA case_sensitive_like (sqlite/dsn.go) — its default LIKE is +// LIKE over raw-cased operands is case-sensitive on both implementations: Dolt uses +// a binary table collation, and SQLite sets PRAGMA case_sensitive_like (sqlite/dsn.go). +// SQLite's default LIKE is // ASCII-case-insensitive and silently diverged (bd-oyvc2.10). IDPrefix filtering // (sqlbuild/filter.go `id LIKE ?`) must therefore distinguish test-AB from test-ab. func testAuditSearchIDPrefixCaseSensitive(t *testing.T, f Factory) { diff --git a/internal/storage/conformance/claim.go b/internal/storage/conformance/claim.go index 0dbc9ca05..5684e5c28 100644 --- a/internal/storage/conformance/claim.go +++ b/internal/storage/conformance/claim.go @@ -2,8 +2,6 @@ package conformance import ( "errors" - "fmt" - "sync" "testing" "time" @@ -12,10 +10,10 @@ import ( "github.com/steveyegge/beads/internal/types" ) -// Claim / lease behavior (Gas Station v1.1 dead-worker recovery). Every backend +// Claim / lease behavior (Gas Station v1.1 dead-worker recovery). Each backend // routes ClaimIssue/ClaimReadyIssue/HeartbeatIssue/ReclaimExpiredLeases through the -// shared issueops implementations, so these assertions hold identically on the Dolt -// reference and every SQL backend. issueops.WithLeaseTTL pins the lease deterministically +// shared issueops implementations, so these assertions hold identically on Dolt and +// SQLite. issueops.WithLeaseTTL pins the lease deterministically // so the reclaim path is testable without wall-clock waits. // testClaim: claiming an open issue stamps assignee, in_progress, started_at, and a @@ -156,102 +154,6 @@ func issueID(i *types.Issue) string { return i.ID } -// requireMultiWriter skips the caller unless the backend backs concurrent writers on -// separate connections. Postgres and MySQL use a multi-connection pool; SQLite pins its -// pool to a single connection (sqliteDialect.Open sets MaxOpenConns(1)) and embedded-Dolt -// serializes writers, so a "concurrent" claim race there only ever exercises the trivial -// serialized path. The dialect name is the profile signal: sqlkit-backed SQL stores -// expose DialectName(); the Dolt reference does not, so it skips via the type assertion. -func requireMultiWriter(t *testing.T, s storage.DoltStorage) { - t.Helper() - dn, ok := s.(interface{ DialectName() string }) - if !ok { - t.Skip("backend does not expose a SQL dialect; concurrent multi-writer claim race not applicable") - } - switch dn.DialectName() { - case "postgres", "mysql": - // multi-connection pool: a genuine cross-connection race. - default: - t.Skipf("dialect %q is single-writer (pool pinned to one connection); concurrent claim race not applicable", dn.DialectName()) - } -} - -// testClaimReadyIssueConcurrentExclusivity races many workers, each on its own pooled -// connection, to claim a small pool of ready issues and asserts every issue is claimed by -// exactly one worker. This is the multi-writer property the SQL backends exist for — the -// whole reason to run bd on Postgres/MySQL instead of single-writer Dolt — and the shared -// claim path's cross-process exclusivity (a conditional-UPDATE CAS: only the writer whose -// UPDATE finds the row still unassigned wins; the losers' ClaimReadyIssue retries the next -// ready candidate) is otherwise only asserted, never raced. Gated to the multi-connection -// backends; see requireMultiWriter. -func testClaimReadyIssueConcurrentExclusivity(t *testing.T, f Factory) { - s := f(t) - requireMultiWriter(t, s) - - const readyCount = 8 - for i := 0; i < readyCount; i++ { - must(t, s.CreateIssue(ctx(), withDefaults(&types.Issue{ - ID: fmt.Sprintf("ccx-%d", i), - Title: "ready", - Priority: 1, - }), "a")) - } - - const workers = 24 // > readyCount, so workers must contend and most must lose a race - - var ( - mu sync.Mutex - claims = make(map[string]string) // issueID -> claiming worker - dupes []string - errs []error - ) - start := make(chan struct{}) - var wg sync.WaitGroup - for w := 0; w < workers; w++ { - wg.Add(1) - go func(worker string) { - defer wg.Done() - <-start // barrier: release all workers together to maximize contention - // Drain: keep claiming until no ready work remains. Bounded so a bug that - // kept returning issues can never hang the suite. - for attempt := 0; attempt < readyCount+workers; attempt++ { - claimed, err := s.ClaimReadyIssue(ctx(), types.WorkFilter{}, worker) - if err != nil { - mu.Lock() - errs = append(errs, err) - mu.Unlock() - return - } - if claimed == nil { - return // no ready work left to claim - } - if claimed.Assignee != worker || claimed.Status != types.StatusInProgress { - t.Errorf("claimed %s not owned by claimer: assignee=%q status=%q", claimed.ID, claimed.Assignee, claimed.Status) - } - mu.Lock() - if prev, ok := claims[claimed.ID]; ok { - dupes = append(dupes, fmt.Sprintf("%s claimed by both %s and %s", claimed.ID, prev, worker)) - } else { - claims[claimed.ID] = worker - } - mu.Unlock() - } - }(fmt.Sprintf("worker-%d", w)) - } - close(start) - wg.Wait() - - for _, e := range errs { - t.Errorf("ClaimReadyIssue errored under concurrency: %v", e) - } - for _, d := range dupes { - t.Errorf("claim exclusivity violated — %s", d) - } - if len(claims) != readyCount { - t.Errorf("distinct claimed issues = %d, want %d (every ready issue claimed exactly once, none lost)", len(claims), readyCount) - } -} - // testHeartbeatRenewsLease: a heartbeat extends the lease (and keeps the claim). // Heartbeating with a one-hour TTL must push lease_expires_at far into the future. func testHeartbeatRenewsLease(t *testing.T, f Factory) { diff --git a/internal/storage/conformance/conformance.go b/internal/storage/conformance/conformance.go index 242e3ea87..9c625e2f3 100644 --- a/internal/storage/conformance/conformance.go +++ b/internal/storage/conformance/conformance.go @@ -149,7 +149,6 @@ func RunAll(t *testing.T, factory Factory) { t.Run("ClaimNotClaimable", func(t *testing.T) { testClaimNotClaimable(t, factory) }) t.Run("ClaimReadyIssue", func(t *testing.T) { testClaimReadyIssue(t, factory) }) t.Run("ClaimReadyIssueLabelFilters", func(t *testing.T) { testClaimReadyIssueLabelFilters(t, factory) }) - t.Run("ClaimReadyIssueConcurrentExclusivity", func(t *testing.T) { testClaimReadyIssueConcurrentExclusivity(t, factory) }) t.Run("HeartbeatRenewsLease", func(t *testing.T) { testHeartbeatRenewsLease(t, factory) }) t.Run("HeartbeatWisp", func(t *testing.T) { testHeartbeatWisp(t, factory) }) t.Run("ReclaimExpiredLease", func(t *testing.T) { testReclaimExpiredLease(t, factory) }) @@ -191,12 +190,10 @@ func RunAll(t *testing.T, factory Factory) { t.Run("Transaction", func(t *testing.T) { testTransaction(t, factory) }) } -// RunDeferredReads runs the subset of the suite covering the shared "deferred" -// non-version-control reads — statistics, external-ref lookup, and staleness — that -// the SQL-family backends (postgres/mysql/sqlite) implement through issueops. RunAll -// is the full fail-loud measurement (and stays red on genuinely Dolt-only methods -// like slots), so these backends run this focused GREEN gate instead. The Dolt -// reference covers the same cases via RunAll. +// RunDeferredReads runs the subset of the suite covering SQLite's shared +// non-version-control reads: statistics, external-ref lookup, and staleness. RunAll +// remains the full Dolt reference; SQLite runs this focused gate for methods supplied +// by issueops while its Dolt-only methods fail loudly as unsupported. func RunDeferredReads(t *testing.T, factory Factory) { t.Helper() t.Run("Statistics", func(t *testing.T) { testStatistics(t, factory) }) diff --git a/internal/storage/conformance/portable.go b/internal/storage/conformance/portable.go index 7cd0afaba..990520a51 100644 --- a/internal/storage/conformance/portable.go +++ b/internal/storage/conformance/portable.go @@ -12,9 +12,9 @@ import ( "github.com/steveyegge/beads/internal/types" ) -// This file holds the behavior contract for the "portable" non-version-control methods -// that the SQL-family backends (postgres/mysql/sqlite) implement through shared issueops -// helpers — molecule rollups, repo-mtime cache, event/dependency streams, dependent +// This file holds the behavior contract for the portable, non-version-control methods +// that SQLite implements through shared issueops helpers: molecule rollups, repo-mtime +// cache, event/dependency streams, dependent // counts, wisp cascade discovery, comment/audit writes, id rekey, source-repo purge, and // batch create. Each case is validated against the embedded-Dolt reference (the oracle) // and, once a method is wired into a backend, must match it there too. @@ -23,7 +23,7 @@ import ( // ties, tie-broken "current step") is asserted as a set, never positionally. // RunPortableMethods runs the portable-method behavior contract. The Dolt reference runs -// it via RunAll; each SQL backend runs it once the methods are implemented. +// it via RunAll; SQLite runs it for the methods supplied by the shared layer. func RunPortableMethods(t *testing.T, factory Factory) { t.Helper() t.Run("MoleculeProgress", func(t *testing.T) { testGetMoleculeProgress(t, factory) }) diff --git a/internal/storage/dolt/open.go b/internal/storage/dolt/open.go index 7cc664cb7..32f8fe122 100644 --- a/internal/storage/dolt/open.go +++ b/internal/storage/dolt/open.go @@ -48,6 +48,25 @@ func ApplyCLIAutoStart(beadsDir string, cfg *Config) { cfg.AutoStart = resolveAutoStart(true, autoStartCfg, mode) } +// requireDoltBackend keeps metadata-driven callers from bypassing the storage +// factory and interpreting another backend's workspace as Dolt. Removed backend +// identifiers deliberately remain recognizable in metadata so this check can fail +// closed instead of opening a new, empty Dolt database. +func requireDoltBackend(fileCfg *configfile.Config) error { + switch fileCfg.Backend { + case configfile.BackendPostgres, configfile.BackendMySQL: + return fmt.Errorf("configured storage backend %q is no longer supported and cannot be opened as Dolt: %s", fileCfg.Backend, configfile.RemovedBackendDetail(fileCfg.Backend)) + } + if !configfile.IsSupportedBackend(fileCfg.Backend) { + return fmt.Errorf("configured storage backend %q in metadata.json is not recognized and cannot be opened as Dolt; %s", fileCfg.Backend, configfile.BackendNotOpenedGuarantee) + } + backend := fileCfg.GetBackend() + if backend != configfile.BackendDolt { + return fmt.Errorf("configured storage backend %q cannot be opened as Dolt", backend) + } + return nil +} + // NewFromConfig creates a DoltStore based on the metadata.json configuration. // beadsDir is the path to the .beads directory. func NewFromConfig(ctx context.Context, beadsDir string) (*DoltStore, error) { @@ -66,6 +85,9 @@ func NewFromConfigWithCLIOptions(ctx context.Context, beadsDir string, cfg *Conf if fileCfg == nil { fileCfg = configfile.DefaultConfig() } + if err := requireDoltBackend(fileCfg); err != nil { + return nil, err + } // Apply central server config as defaults for any server fields not // set in the per-project metadata.json. This eliminates the need to @@ -93,6 +115,9 @@ func NewFromConfigWithOptions(ctx context.Context, beadsDir string, cfg *Config) if fileCfg == nil { fileCfg = configfile.DefaultConfig() } + if err := requireDoltBackend(fileCfg); err != nil { + return nil, err + } // Apply central server config as defaults for any server fields not // set in the per-project metadata.json. diff --git a/internal/storage/dolt/open_test.go b/internal/storage/dolt/open_test.go index 2e5b0b9af..9a3228d8b 100644 --- a/internal/storage/dolt/open_test.go +++ b/internal/storage/dolt/open_test.go @@ -162,6 +162,50 @@ func TestApplyCLIAutoStart_DisableAutoStartWins(t *testing.T) { } } +func TestConfigConstructorsRejectNonDoltBackends(t *testing.T) { + for _, backend := range []string{ + configfile.BackendSQLite, + configfile.BackendPostgres, + configfile.BackendMySQL, + "mystery", + } { + t.Run(backend, func(t *testing.T) { + beadsDir := t.TempDir() + if err := (&configfile.Config{Backend: backend}).Save(beadsDir); err != nil { + t.Fatalf("save metadata: %v", err) + } + + constructors := map[string]func() (*DoltStore, error){ + "standard": func() (*DoltStore, error) { + return NewFromConfigWithOptions(t.Context(), beadsDir, &Config{DisableAutoStart: true}) + }, + "cli": func() (*DoltStore, error) { + return NewFromConfigWithCLIOptions(t.Context(), beadsDir, &Config{DisableAutoStart: true}) + }, + } + for name, construct := range constructors { + t.Run(name, func(t *testing.T) { + store, err := construct() + if store != nil { + _ = store.Close() + t.Fatalf("non-Dolt backend %q unexpectedly opened as Dolt", backend) + } + if err == nil || !strings.Contains(err.Error(), "cannot be opened as Dolt") { + t.Fatalf("constructor error = %v, want backend mismatch", err) + } + if backend == configfile.BackendPostgres || backend == configfile.BackendMySQL { + for _, want := range []string{"no longer supported", "was not opened or modified", "bd help init-safety"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("removed-backend constructor error missing %q: %v", want, err) + } + } + } + }) + } + }) + } +} + func TestCLIDirUsesSharedDoltRootInSharedServerMode(t *testing.T) { sharedRoot := t.TempDir() t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") diff --git a/internal/storage/embeddeddolt/iter_stubs.go b/internal/storage/embeddeddolt/iter_stubs.go index cb5f76fcb..39f8e9a4b 100644 --- a/internal/storage/embeddeddolt/iter_stubs.go +++ b/internal/storage/embeddeddolt/iter_stubs.go @@ -4,8 +4,8 @@ // // Slice-wrapping stubs for the Iter* methods. The embedded Dolt backend // uses a per-method short-lived connection model (`withConn`) which is -// incompatible with the dedicated-conn cursor pattern used by the -// streaming iterators in internal/storage/dolt and internal/storage/postgres. +// incompatible with the dedicated-connection cursor pattern used by the +// streaming iterators in internal/storage/dolt. // The interface ships complete now (be-jaavsb / be-yinl4d); a follow-up // child of be-yinl4d may add a streaming variant if EmbeddedDoltStore // gains a cursor-conn API. For now every Iter* method materializes the diff --git a/internal/storage/errors_unsupported.go b/internal/storage/errors_unsupported.go index fd4c70359..6867b739a 100644 --- a/internal/storage/errors_unsupported.go +++ b/internal/storage/errors_unsupported.go @@ -7,7 +7,7 @@ import "fmt" // taxonomy; this is the embryo of the seam's ErrUnsupported{Op, Backend}. type ErrUnsupported struct { Op string // method name, e.g. "AddLabel" or "Transaction.CreateIssues" - Backend string // e.g. "postgres" + Backend string // e.g. "sqlite" } func (e *ErrUnsupported) Error() string { diff --git a/internal/storage/issueops/blocked_state.go b/internal/storage/issueops/blocked_state.go index 61090dd9f..79a37065a 100644 --- a/internal/storage/issueops/blocked_state.go +++ b/internal/storage/issueops/blocked_state.go @@ -334,29 +334,6 @@ func unmarkBlockedTemplateForWisps() string { `, waitsForGateBlockedSQL) } -// SelfReferentialRecomputeTemplates returns the four is_blocked recompute UPDATE -// statements exactly as issueops emits them to the driver (with a representative -// IN-clause batch substituted for the placeholder). Each one updates issues/wisps -// while its WHERE clause self-references that same target table in EXISTS subqueries, -// which MySQL 8 rejects with error 1093 unless the mysqldialect driver rewrites the -// statement into a materialized derived-table form. -// -// It is exported solely so that rewrite can be regression-tested against these real -// templates instead of a hand-copied fixture: if a template ever drifts out of the -// shape the rewrite recognizes (a predicate before `id IN`, a backticked table, a -// renamed alias, changed whitespace), the mysqldialect rewrite test fails in a normal -// non-gated `go test` run rather than only under the env-gated MySQL conformance suite. -func SelfReferentialRecomputeTemplates() []string { - placeholders, _ := buildSQLInClause([]string{"a", "b", "c"}) - render := func(tmpl string) string { return fmt.Sprintf(tmpl, placeholders) } - return []string{ - render(markBlockedTemplateForIssues()), - render(unmarkBlockedTemplateForIssues()), - render(markBlockedTemplateForWisps()), - render(unmarkBlockedTemplateForWisps()), - } -} - //nolint:gosec // G201: callers pass constant templates; only IN-clause placeholders are formatted in. func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl, unmarkTmpl string, ids []string) (int64, error) { var changed int64 diff --git a/internal/storage/issueops/count.go b/internal/storage/issueops/count.go index 92c4d9d21..71d7fcd3a 100644 --- a/internal/storage/issueops/count.go +++ b/internal/storage/issueops/count.go @@ -186,9 +186,7 @@ func countByColumnInTx(ctx context.Context, tx *sql.Tx, filter types.IssueFilter whereSQL = " WHERE " + strings.Join(clauses, " AND ") } // CAST the group column to text before COALESCE so an integer column (priority) - // COALESCEs with '' consistently. MySQL/Dolt coerce int->text implicitly, but - // Postgres rejects COALESCE(integer, '') (22P02); CAST(... AS CHAR) is portable - // (pgdialect -> ::text, native on MySQL/SQLite) and byte-identical on every backend. + // COALESCEs with '' consistently across Dolt and SQLite. //nolint:gosec // G201: tables.Main hardcoded; col validated by caller query := fmt.Sprintf("SELECT COALESCE(CAST(%s AS CHAR), ''), COUNT(*) FROM %s%s GROUP BY %s", col, tables.Main, whereSQL, col) rows, err := tx.QueryContext(ctx, query, args...) diff --git a/internal/storage/issueops/create.go b/internal/storage/issueops/create.go index 1803e7dd4..cf77aa61f 100644 --- a/internal/storage/issueops/create.go +++ b/internal/storage/issueops/create.go @@ -905,10 +905,9 @@ func ReconcileChildCounters(ctx context.Context, tx *sql.Tx, issues []*types.Iss if err == nil && current >= b.maxChild { continue } - // Qualify the existing-row column with the table name. Bare `last_child` - // on the update RHS is a valid MySQL "current row value" reference, but - // Postgres sees it as ambiguous between the target row and EXCLUDED - // (SQLSTATE 42702); table-qualifying it is unambiguous on every backend. + // Qualify the existing-row column with the table name so the canonical + // MySQL form and SQLite's translated ON CONFLICT form both unambiguously + // refer to the target row rather than the incoming value. //nolint:gosec // G201: table is one of two hardcoded constants. if _, err := tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO %[1]s (parent_id, last_child) VALUES (?, ?) diff --git a/internal/storage/issueops/helpers.go b/internal/storage/issueops/helpers.go index 86d1e584d..bebd719f5 100644 --- a/internal/storage/issueops/helpers.go +++ b/internal/storage/issueops/helpers.go @@ -89,10 +89,9 @@ func issueUpsertAssignments(table string, rejectStaleUpdate bool) string { assignments := make([]string, 0, len(issueUpsertColumns)) for _, col := range issueUpsertColumns { if rejectStaleUpdate { - // Qualify the existing-row references with the table name. Postgres's - // ON CONFLICT DO UPDATE rejects a bare `updated_at` as ambiguous (it could - // be the target row or EXCLUDED); .updated_at is unambiguous and is - // also accepted by MySQL/Dolt/SQLite. VALUES(...) is the incoming row. + // Qualify existing-row references with the table name so the target value + // remains unambiguous after the SQLite upsert translation. VALUES(...) is + // the incoming row in the canonical Dolt/MySQL-dialect statement. assignments = append(assignments, fmt.Sprintf("%s = IF(VALUES(updated_at) > %s.updated_at, VALUES(%s), %s.%s)", col, table, col, table, col)) } else { diff --git a/internal/storage/iter.go b/internal/storage/iter.go index 365ff27ab..848a593a7 100644 --- a/internal/storage/iter.go +++ b/internal/storage/iter.go @@ -2,7 +2,7 @@ // // Iter[T] is a generic streaming-iterator shape that lets storage callers // walk an unbounded result set without materializing the whole slice. -// Per-method implementations live in the dolt/ and postgres/ sub-packages. +// Per-method implementations live in the concrete backend packages. // // See be-jaavsb (design) and be-yinl4d (architecture) for the motivation. package storage diff --git a/internal/storage/mysql/completeness_test.go b/internal/storage/mysql/completeness_test.go deleted file mode 100644 index 7fd092bed..000000000 --- a/internal/storage/mysql/completeness_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package mysql - -import ( - "os" - "regexp" - "sort" - "testing" - - "github.com/steveyegge/beads/internal/storage/conformance" -) - -// legitimatelyUnsupported is the EXPLICIT denominator of every storage.DoltStorage -// method the Postgres wedge deliberately does not implement, each with a reason. -// -// The interface-completeness gate (TestInterfaceCompleteness) asserts the -// generated capability shell (unsupported_gen.go) contains EXACTLY these methods -// — no more, no less. This is the guard the red-team had to stand in for: a -// method that silently resolves to the typed-unsupported shell (the class that -// bit us: DeleteIssues, GetCustomStatuses, the batch reads, …) fails this test -// at build time instead of failing a user at runtime. Adding a method to the -// shell without an entry here is a hard error that forces the triage question: -// "implement it in sqlkit, or record here why the wedge legitimately omits it." -var legitimatelyUnsupported = map[string]string{ - // VersionControl — the Dolt commit graph. The wedge's whole premise is - // "bd on Postgres, history/version-control simply absent". - "Branch": "VC: Dolt branch", "Checkout": "VC: Dolt checkout", - "CurrentBranch": "VC", "DeleteBranch": "VC", "ListBranches": "VC", - // Commit/CommitWithConfig/CommitMergeResolution are NO-OPS on Postgres (data - // is already durable per-tx), not unsupported — implemented on postgres.Store. - "CommitExists": "VC", "GetCurrentCommit": "VC: Dolt hash", "Status": "VC", - "Log": "VC: dolt_log", "Merge": "VC", "GetConflicts": "VC", "ResolveConflicts": "VC", - - // HistoryViewer — Dolt time-travel. - "History": "history: dolt_log", "AsOf": "history: as-of ref", "Diff": "history: dolt_diff", - - // RemoteStore — Dolt remotes. - "AddRemote": "remote", "RemoveRemote": "remote", "HasRemote": "remote", "ListRemotes": "remote", - "Push": "remote", "Pull": "remote", "ForcePush": "remote", "PushRemote": "remote", - "PullRemote": "remote", "Fetch": "remote", "PushTo": "remote", "PullFrom": "remote", - - // SyncStore — Dolt peer sync. - "Sync": "sync", "SyncStatus": "sync", - - // FederationStore — Dolt-remote-backed peer table. - "AddFederationPeer": "federation", "GetFederationPeer": "federation", - "ListFederationPeers": "federation", "RemoveFederationPeer": "federation", - - // CompactionStore — bound to Dolt commit hashes. - "CheckEligibility": "compaction", "ApplyCompaction": "compaction", - "GetTier1Candidates": "compaction", "GetTier2Candidates": "compaction", - "SnapshotIssue": "compaction", "GetCompactionSnapshot": "compaction", "RestoreFromSnapshot": "compaction", - - // MergeSlot — the Dolt merge-serialization primitive. - "MergeSlotCreate": "merge-slot", "MergeSlotCheck": "merge-slot", - "MergeSlotAcquire": "merge-slot", "MergeSlotRelease": "merge-slot", -} - -var shellMethodRe = regexp.MustCompile(`func \(unsupportedDoltStorage\) ([A-Za-z0-9]+)\(`) - -// TestInterfaceCompleteness is the interface-completeness gate: the generated -// shell must equal the audited legitimatelyUnsupported set, both directions. -func TestInterfaceCompleteness(t *testing.T) { - src, err := os.ReadFile("unsupported_gen.go") - if err != nil { - t.Fatalf("read unsupported_gen.go: %v", err) - } - shell := map[string]bool{} - for _, m := range shellMethodRe.FindAllStringSubmatch(string(src), -1) { - shell[m[1]] = true - } - if len(shell) == 0 { - t.Fatal("parsed 0 shell methods — regex or file drift") - } - - // (1) No SILENT gap: every shell method must be explicitly justified. - var unjustified []string - for m := range shell { - if _, ok := legitimatelyUnsupported[m]; !ok { - unjustified = append(unjustified, m) - } - } - sort.Strings(unjustified) - for _, m := range unjustified { - t.Errorf("method %q resolves to the typed-unsupported shell but is NOT in legitimatelyUnsupported: implement it on *sqlkit.Store, or add it here with a reason", m) - } - - // (2) No STALE allowlist: every justified method must still be in the shell - // (else it was implemented and the entry should be removed). - var stale []string - for m := range legitimatelyUnsupported { - if !shell[m] { - stale = append(stale, m) - } - } - sort.Strings(stale) - for _, m := range stale { - t.Errorf("method %q is in legitimatelyUnsupported but no longer in the shell (implemented?): remove the allowlist entry", m) - } -} - -// TestUnsupportedContract is the behavioral complement to TestInterfaceCompleteness: -// every allowlisted method must actually return a typed storage.ErrUnsupported when -// called, not panic or return a different error. DB-free — the generated stubs ignore -// their receiver, so a zero-value store answers them. -func TestUnsupportedContract(t *testing.T) { - conformance.RunUnsupportedContract(t, &Store{}, legitimatelyUnsupported) -} diff --git a/internal/storage/mysql/conformance_test.go b/internal/storage/mysql/conformance_test.go deleted file mode 100644 index e7946f517..000000000 --- a/internal/storage/mysql/conformance_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package mysql - -import ( - "context" - "database/sql" - "fmt" - "os" - "sync/atomic" - "testing" - "time" - - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/storage/conformance" -) - -// TestConformance runs bd's backend-agnostic storage conformance suite against the -// MySQL backend. Every failure is a MySQL gap: an allowlisted-unsupported method -// (expected — completeness_test.go) or a latent divergence. Gated on -// BEADS_MYSQL_TEST_URL (a server DSN, e.g. user:pass@tcp(host:3306)/). -func TestConformance(t *testing.T) { - url := os.Getenv("BEADS_MYSQL_TEST_URL") - if url == "" { - t.Skip("BEADS_MYSQL_TEST_URL not set") - } - conformance.RunAll(t, mysqlConformanceFactory(url)) -} - -// TestDeferredReads runs the shared deferred non-version-control reads (statistics, -// external-ref, stale) that the backend now implements through issueops, as a GREEN -// gate. RunAll stays the fail-loud measurement (red on genuinely Dolt-only methods -// like slots), so this focused gate is what conformance.sh runs. Gated on -// BEADS_MYSQL_TEST_URL. -func TestDeferredReads(t *testing.T) { - url := os.Getenv("BEADS_MYSQL_TEST_URL") - if url == "" { - t.Skip("BEADS_MYSQL_TEST_URL not set") - } - conformance.RunDeferredReads(t, mysqlConformanceFactory(url)) -} - -// TestPortableMethods runs the full portable-method behavior contract. Red until the -// methods are wired into sqlkit; green after. Gated on BEADS_MYSQL_TEST_URL. -func TestPortableMethods(t *testing.T) { - url := os.Getenv("BEADS_MYSQL_TEST_URL") - if url == "" { - t.Skip("BEADS_MYSQL_TEST_URL not set") - } - conformance.RunPortableMethods(t, mysqlConformanceFactory(url)) -} - -// mysqlConformanceFactory provisions a fresh, isolated database per sub-test against -// the MySQL server at url, seeded with issue_prefix as `bd init` leaves it. -func mysqlConformanceFactory(url string) conformance.Factory { - base := time.Now().UnixNano() - var seq int64 - return func(t *testing.T) storage.DoltStorage { - ctx := context.Background() - database := fmt.Sprintf("conf_%d_%d", base, atomic.AddInt64(&seq, 1)) - - st, err := Provision(ctx, url, database) - if err != nil { - t.Fatalf("Provision: %v", err) - } - // The conformance factory contract wants an init'd store (issue_prefix set), - // as `bd init` leaves it. No Commit — MySQL has no commit graph. - if err := st.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("SetConfig(issue_prefix): %v", err) - } - t.Cleanup(func() { - if serverDSN, e := withDatabase(url, ""); e == nil { - if srv, e2 := sql.Open("mysql", serverDSN); e2 == nil { - _, _ = srv.ExecContext(context.Background(), "DROP DATABASE IF EXISTS `"+database+"`") - _ = srv.Close() - } - } - _ = st.Close() - }) - return st - } -} diff --git a/internal/storage/mysql/credential.go b/internal/storage/mysql/credential.go deleted file mode 100644 index 7026176c3..000000000 --- a/internal/storage/mysql/credential.go +++ /dev/null @@ -1,67 +0,0 @@ -package mysql - -import ( - "context" - "fmt" - "os" - - "github.com/steveyegge/beads/internal/configfile" - "github.com/steveyegge/beads/internal/creds" - "github.com/steveyegge/beads/internal/storage/mysqldialect" -) - -// resolveDSNCredential returns baseDSN with a credential resolved and placed, ready to -// connect. The precedence, high to low: -// -// - BEADS_MYSQL_CREDENTIAL_COMMAND set is a hard error: an identity credential is -// presented as a username to a gateway, and this backend connects directly; -// - the DSN already carries a password (a full BEADS_MYSQL_URL, or an explicit -// --mysql-url) — it wins outright, no ladder is applied; -// - BEADS_MYSQL_PASSWORD_COMMAND — a credential command (e.g. Vault / RDS-IAM / GCP-IAM); -// run at open time, cached until near expiry; -// - BEADS_MYSQL_PASSWORD — a static password; -// - the credentials file [host:port] ($BEADS_CREDENTIALS_FILE or -// ~/.config/beads/credentials); -// - nothing configured — baseDSN is returned untouched and the connection is -// attempted with an empty password (valid only for an unsecured dev server; -// go-sql-driver reads no env vars or option files, so there is no ~/.my.cnf -// fallback). -// -// It fails closed: if a configured credential command errors, the open aborts — it -// never silently downgrades to a lower rung. -func resolveDSNCredential(ctx context.Context, cfg *configfile.Config, baseDSN string) (string, error) { - if os.Getenv("BEADS_MYSQL_CREDENTIAL_COMMAND") != "" { - return "", fmt.Errorf("mysql: BEADS_MYSQL_CREDENTIAL_COMMAND is reserved: an identity credential is presented as a username to a gateway, and the mysql backend connects directly to the database with no gateway to present it to; use BEADS_MYSQL_PASSWORD_COMMAND for a helper that produces a password") - } - // host:port keys the credentials-file lookup; ok=false (unix socket / unparseable) - // leaves the file rung not-configured (zero Host/Port), never failing. - host, port, _ := mysqldialect.HostPort(baseDSN) - return resolveDSNWithSources(ctx, baseDSN, - creds.CommandSource{Command: cfg.GetMySQLPasswordCommand(), Kind: creds.KindSecret, Label: "BEADS_MYSQL_PASSWORD_COMMAND"}, - creds.EnvSource{Var: "BEADS_MYSQL_PASSWORD"}, - creds.FileSource{Host: host, Port: port, Lookup: configfile.LookupCredentialsPassword, Label: "credentials-file"}, - ) -} - -// resolveDSNWithSources is the testable core of resolveDSNCredential: it applies the -// given credential ladder to baseDSN. -func resolveDSNWithSources(ctx context.Context, baseDSN string, sources ...creds.Source) (string, error) { - if mysqldialect.HasPassword(baseDSN) { - return baseDSN, nil - } - cred, ok, err := creds.ResolveLadder(ctx, sources...) - if err != nil { - return "", err - } - if !ok { - return baseDSN, nil - } - // The SQL backends take a secret in the password slot; an identity credential - // (presented as a username to a gateway) has no home on a direct connection, so - // refuse it loudly rather than land a token where a password belongs. Defense in - // depth behind the BEADS_MYSQL_CREDENTIAL_COMMAND reject above. - if cred.Kind != creds.KindSecret { - return "", fmt.Errorf("mysql: credential from %s is an identity, not a password; the mysql backend connects directly and has no gateway to present it to", cred.Source) - } - return mysqldialect.WithCredential(baseDSN, cred.Username, cred.Value) -} diff --git a/internal/storage/mysql/credential_test.go b/internal/storage/mysql/credential_test.go deleted file mode 100644 index 5d488a0ba..000000000 --- a/internal/storage/mysql/credential_test.go +++ /dev/null @@ -1,288 +0,0 @@ -package mysql - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - - gomysql "github.com/go-sql-driver/mysql" - "github.com/steveyegge/beads/internal/configfile" - "github.com/steveyegge/beads/internal/creds" -) - -const baseDSN = "bts@tcp(127.0.0.1:55441)/" - -// TestMain points the credentials-file rung at a nonexistent path in a private temp -// dir so tests never read the developer's real ~/.config/beads/credentials. -// go-sql-driver reads no ambient env/option files, so there is nothing else to -// neutralize. Tests that exercise the file rung override it per-test with t.Setenv. -func TestMain(m *testing.M) { - dir, err := os.MkdirTemp("", "beads-mysql-cred-test") - if err != nil { - panic(err) - } - os.Setenv("BEADS_CREDENTIALS_FILE", filepath.Join(dir, "none")) - code := m.Run() - _ = os.RemoveAll(dir) - os.Exit(code) -} - -// password parses out and returns the password go-sql-driver sees. -func password(t *testing.T, dsn string) string { - t.Helper() - cfg, err := gomysql.ParseDSN(dsn) - if err != nil { - t.Fatalf("gomysql.ParseDSN(%q): %v", dsn, err) - } - return cfg.Passwd -} - -// user parses out and returns the username go-sql-driver sees. -func user(t *testing.T, dsn string) string { - t.Helper() - cfg, err := gomysql.ParseDSN(dsn) - if err != nil { - t.Fatalf("gomysql.ParseDSN(%q): %v", dsn, err) - } - return cfg.User -} - -// writeCredsFile writes an INI credentials file and returns its path. -func writeCredsFile(t *testing.T, body string) string { - t.Helper() - p := filepath.Join(t.TempDir(), "credentials") - if err := os.WriteFile(p, []byte(body), 0o600); err != nil { - t.Fatal(err) - } - return p -} - -func TestResolveDSNCredentialCommand(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "printf cmd-pw") - t.Setenv("BEADS_MYSQL_PASSWORD", "") - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "cmd-pw" { - t.Fatalf("password = %q, want cmd-pw", pw) - } -} - -func TestResolveDSNCredentialEnv(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") - t.Setenv("BEADS_MYSQL_PASSWORD", "env-pw") - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "env-pw" { - t.Fatalf("password = %q, want env-pw", pw) - } -} - -// The command out-ranks the static env password. -func TestResolveDSNCredentialCommandBeatsEnv(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "printf cmd-wins") - t.Setenv("BEADS_MYSQL_PASSWORD", "env-loses") - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "cmd-wins" { - t.Fatalf("password = %q, want cmd-wins", pw) - } -} - -// A configured-but-erroring command aborts the open and does NOT fall through to the -// env password. Here a bare token with whitespace is rejected by parseCredential — a -// configured error, not a command exit failure. -func TestResolveDSNCredentialFailsClosed(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", `printf 'access denied'`) - t.Setenv("BEADS_MYSQL_PASSWORD", "env-would-be-wrong") - _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err == nil { - t.Fatal("expected an error (fail-closed); a broken command must not fall through to BEADS_MYSQL_PASSWORD") - } -} - -// A command that exits non-zero also aborts (the other configured-error shape). -func TestResolveDSNCredentialFailsClosedOnExit(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "false") - t.Setenv("BEADS_MYSQL_PASSWORD", "env-would-be-wrong") - _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err == nil { - t.Fatal("expected an error when the command exits non-zero") - } -} - -// A getToken/ExecCredential JSON envelope resolves the password end-to-end. -func TestResolveDSNCredentialJSONEnvelope(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", `printf '{"access_token":"tok-pw","expires_in":90}'`) - t.Setenv("BEADS_MYSQL_PASSWORD", "") - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "tok-pw" { - t.Fatalf("password = %q, want tok-pw", pw) - } -} - -// End-to-end: resolving a password into a userless base DSN must fail loudly (the -// grammar would silently drop it), not connect passwordless. -func TestResolveDSNCredentialUserlessDSNFailsLoudly(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") - t.Setenv("BEADS_MYSQL_PASSWORD", "some-pw") - _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, "tcp(127.0.0.1:55441)/") - if err == nil { - t.Fatal("expected an error placing a password into a userless DSN") - } -} - -// withDatabase refuses a userless password-bearing DSN (the path that skips the -// ladder via an inline ":secret@tcp(host)/" URL) rather than silently dropping it. -func TestWithDatabaseRefusesUserlessPassword(t *testing.T) { - if _, err := withDatabase(":secret@tcp(127.0.0.1:55441)/", "db"); err == nil { - t.Fatal("expected withDatabase to refuse a userless password-bearing DSN") - } - // A normal user:pass DSN still works. - if _, err := withDatabase("bts:bts@tcp(127.0.0.1:55441)/", "db"); err != nil { - t.Fatalf("withDatabase rejected a valid DSN: %v", err) - } -} - -func TestResolveDSNCredentialNothingConfigured(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") - t.Setenv("BEADS_MYSQL_PASSWORD", "") - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if got != baseDSN { - t.Fatalf("dsn = %q, want it unchanged (%q)", got, baseDSN) - } -} - -func TestResolveDSNCredentialExistingPasswordWins(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "printf should-not-run") - t.Setenv("BEADS_MYSQL_PASSWORD", "should-not-run") - withPw := "bts:already-here@tcp(127.0.0.1:55441)/" - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, withPw) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "already-here" { - t.Fatalf("password = %q, want already-here (existing password must win)", pw) - } -} - -// The credentials file resolves the password when no command or env is set. -func TestResolveDSNCredentialFile(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") - t.Setenv("BEADS_MYSQL_PASSWORD", "") - t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:55441]\npassword=file-pw\n")) - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "file-pw" { - t.Fatalf("password = %q, want file-pw", pw) - } -} - -// The env password out-ranks the credentials file. -func TestResolveDSNCredentialEnvBeatsFile(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") - t.Setenv("BEADS_MYSQL_PASSWORD", "env-pw") - t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:55441]\npassword=file-pw\n")) - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "env-pw" { - t.Fatalf("password = %q, want env-pw (env out-ranks file)", pw) - } -} - -// A broken command fails closed even when the file could supply a valid password. -func TestResolveDSNCredentialBrokenCommandNotRescuedByFile(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "false") - t.Setenv("BEADS_MYSQL_PASSWORD", "") - t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:55441]\npassword=file-pw\n")) - if _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN); err == nil { - t.Fatal("expected fail-closed; the credentials file must not rescue a broken command") - } -} - -// A unix-socket DSN has no [host:port] endpoint, so the file rung degrades to -// not-configured (no crash), leaving the DSN untouched. -func TestResolveDSNCredentialFileUnixSocketDegrades(t *testing.T) { - t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") - t.Setenv("BEADS_MYSQL_PASSWORD", "") - t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:55441]\npassword=file-pw\n")) - unixDSN := "bts@unix(/tmp/mysql.sock)/db" - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, unixDSN) - if err != nil { - t.Fatal(err) - } - if got != unixDSN { - t.Fatalf("dsn = %q, want it unchanged (unix socket has no file-rung endpoint)", got) - } -} - -// A set BEADS_MYSQL_CREDENTIAL_COMMAND is a hard error, pre-empting even a -// password-bearing DSN and any lower rung. -func TestResolveDSNCredentialRejectsReservedIdentityVar(t *testing.T) { - t.Setenv("BEADS_MYSQL_CREDENTIAL_COMMAND", "get-credential") - t.Setenv("BEADS_MYSQL_PASSWORD", "would-otherwise-work") - _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, "bts:has-pw@tcp(127.0.0.1:55441)/") - if err == nil || !strings.Contains(err.Error(), "BEADS_MYSQL_CREDENTIAL_COMMAND") { - t.Fatalf("expected a reserved-var rejection naming BEADS_MYSQL_CREDENTIAL_COMMAND, got %v", err) - } -} - -// staticSource is a test Source with a fixed credential, used to exercise placement -// and the identity-refusal guard directly. -type staticSource struct{ cred creds.Credential } - -func (s staticSource) Name() string { return "test" } -func (s staticSource) Resolve(context.Context) (creds.Credential, bool, error) { - return s.cred, true, nil -} - -func TestResolveDSNCredentialRefusesIdentity(t *testing.T) { - src := staticSource{cred: creds.Credential{Value: "an-identity-token", Kind: creds.KindIdentity, Source: "test"}} - _, err := resolveDSNWithSources(context.Background(), baseDSN, src) - if err == nil || !strings.Contains(err.Error(), "identity") { - t.Fatalf("expected an identity-refusal error, got %v", err) - } -} - -// A credential carrying a username (a Vault dynamic user/password pair) places both, -// overriding the base DSN's user. -func TestResolveDSNCredentialPlacesUsername(t *testing.T) { - src := staticSource{cred: creds.Credential{Value: "pw", Username: "dyn", Kind: creds.KindSecret}} - got, err := resolveDSNWithSources(context.Background(), baseDSN, src) - if err != nil { - t.Fatal(err) - } - if u, pw := user(t, got), password(t, got); u != "dyn" || pw != "pw" { - t.Fatalf("(user,password) = (%q,%q), want (dyn,pw)", u, pw) - } -} - -// A dynamic username satisfies the userless-DSN requirement: a bare tcp(host)/ base -// plus a user/password pair is a valid configuration. -func TestResolveDSNWithSourcesUsernameSatisfiesUserlessBase(t *testing.T) { - src := staticSource{cred: creds.Credential{Value: "pw", Username: "dyn", Kind: creds.KindSecret}} - got, err := resolveDSNWithSources(context.Background(), "tcp(127.0.0.1:55441)/", src) - if err != nil { - t.Fatal(err) - } - if u, pw := user(t, got), password(t, got); u != "dyn" || pw != "pw" { - t.Fatalf("(user,password) = (%q,%q), want (dyn,pw)", u, pw) - } -} diff --git a/internal/storage/mysql/dialect.go b/internal/storage/mysql/dialect.go deleted file mode 100644 index 24ef3200b..000000000 --- a/internal/storage/mysql/dialect.go +++ /dev/null @@ -1,23 +0,0 @@ -package mysql - -import ( - "context" - "database/sql" - - "github.com/steveyegge/beads/internal/storage/mysqldialect" -) - -// mysqlDialect is the MySQL SQL flavor. bd's canonical SQL is already MySQL, so this -// is nearly the identity dialect: mysqldialect.Open wraps go-sql-driver with a single -// rewrite (the self-referential is_blocked recompute UPDATE that MySQL 8 rejects) and -// passes everything else through verbatim. The dsn already targets the workspace -// database. -type mysqlDialect struct { - dsn string -} - -func (d mysqlDialect) Name() string { return "mysql" } - -func (d mysqlDialect) Open(_ context.Context) (*sql.DB, error) { - return mysqldialect.Open(d.dsn) -} diff --git a/internal/storage/mysql/dsn.go b/internal/storage/mysql/dsn.go deleted file mode 100644 index 5097e549a..000000000 --- a/internal/storage/mysql/dsn.go +++ /dev/null @@ -1,53 +0,0 @@ -package mysql - -import ( - "fmt" - - gomysql "github.com/go-sql-driver/mysql" -) - -// RedactPassword returns dsn with the password removed, or an error if it cannot be -// stripped. It uses go-sql-driver's own parser (never hand-rolled string surgery) and -// verifies with a re-parse that no password survives, failing closed. bd init persists -// the returned string to metadata.json; the password is re-supplied at open time by -// the credential ladder (see resolveDSNCredential in credential.go). -func RedactPassword(dsn string) (string, error) { - cfg, err := gomysql.ParseDSN(dsn) - if err != nil { - return "", fmt.Errorf("parse DSN: %w", err) - } - cfg.Passwd = "" - stripped := cfg.FormatDSN() - if again, err := gomysql.ParseDSN(stripped); err != nil || again.Passwd != "" { - return "", fmt.Errorf("refusing to persist a connection string that still carries a password; supply it via BEADS_MYSQL_PASSWORD instead of embedding it in --mysql-url") - } - return stripped, nil -} - -// withDatabase returns dsn with its default database set to database (or cleared when -// database is ""). Used to derive a server connection (for CREATE DATABASE) and the -// per-workspace connection from one base DSN. It also forces parseTime and UTC so -// datetime columns round-trip identically to the Dolt reference. -func withDatabase(dsn, database string) (string, error) { - cfg, err := gomysql.ParseDSN(dsn) - if err != nil { - return "", err - } - // The go-sql-driver grammar cannot carry a password without a username — - // FormatDSN would silently drop it. Refuse loudly rather than reformat this DSN - // into a passwordless one and connect (the same invariant mysqldialect.WithPassword - // enforces; this covers the path where a userless password-bearing DSN reaches us - // directly, e.g. an inline ":secret@tcp(host)/" URL that skips the credential ladder). - if cfg.User == "" && cfg.Passwd != "" { - return "", fmt.Errorf("connection string carries a password but no username; the MySQL DSN grammar cannot carry a password without a user (it would be silently dropped) — add a user to the DSN") - } - cfg.DBName = database - cfg.ParseTime = true - if cfg.Params == nil { - cfg.Params = map[string]string{} - } - // Pin the session time zone to UTC so CURRENT_TIMESTAMP defaults and reads match - // the shared layer's UTC assumption (and the Dolt reference). - cfg.Params["time_zone"] = "'+00:00'" - return cfg.FormatDSN(), nil -} diff --git a/internal/storage/mysql/fromconfig.go b/internal/storage/mysql/fromconfig.go deleted file mode 100644 index 4ee3d5112..000000000 --- a/internal/storage/mysql/fromconfig.go +++ /dev/null @@ -1,57 +0,0 @@ -package mysql - -import ( - "context" - "fmt" - - "github.com/steveyegge/beads/internal/configfile" - "github.com/steveyegge/beads/internal/storage" -) - -// NewFromConfig opens the MySQL backend for a workspace. It reads the base server DSN -// and per-workspace database from .beads/metadata.json, resolves the password through -// the credential ladder (command > env > credentials file, fail-closed) and places it into the DSN — -// never persisting it — then provisions and returns the store. It is the factory arm -// cmd/bd dispatches to when metadata.json has backend="mysql". -func NewFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { - cfg, err := configfile.Load(beadsDir) - if err != nil { - return nil, fmt.Errorf("mysql: load config: %w", err) - } - dsn := cfg.GetMySQLDSN() - if dsn == "" { - return nil, fmt.Errorf("mysql: no DSN (set mysql_dsn in metadata.json, or BEADS_MYSQL_URL)") - } - dsn, err = resolveDSNCredential(ctx, cfg, dsn) - if err != nil { - return nil, fmt.Errorf("mysql: resolve credential: %w", err) - } - database := cfg.GetMySQLDatabase() - if database == "" { - return nil, fmt.Errorf("mysql: no database (set mysql_database in metadata.json)") - } - return Provision(ctx, dsn, database) -} - -// Provision opens the MySQL backend from an explicit server DSN and workspace -// database: it creates the database if absent and applies the schema (InitSchema — -// idempotent, config seeds on first provision only), then returns the store over the -// per-workspace connection. baseDSN may carry a password; it is used only to connect -// and is never persisted. bd init calls this directly; NewFromConfig calls it after -// resolving DSN/database from metadata. -func Provision(ctx context.Context, baseDSN, database string) (storage.DoltStorage, error) { - if baseDSN == "" { - return nil, fmt.Errorf("mysql: empty DSN") - } - if database == "" { - return nil, fmt.Errorf("mysql: empty database") - } - if err := InitSchema(ctx, baseDSN, database); err != nil { - return nil, err - } - wsDSN, err := withDatabase(baseDSN, database) - if err != nil { - return nil, fmt.Errorf("mysql: parse DSN: %w", err) - } - return New(ctx, Config{DSN: wsDSN}) -} diff --git a/internal/storage/mysql/redact_test.go b/internal/storage/mysql/redact_test.go deleted file mode 100644 index 4bd55a76e..000000000 --- a/internal/storage/mysql/redact_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package mysql - -import ( - "testing" - - gomysql "github.com/go-sql-driver/mysql" -) - -// placeholderPW is an obvious non-secret stand-in used wherever a password appears in -// these tests, so nothing here can be mistaken for a leaked real credential. -const placeholderPW = "s3cr3t-PLACEHOLDER" - -// TestRedactPassword locks the security invariant for mysql.RedactPassword across every -// secret-bearing DSN shape the go-sql-driver grammar accepts. For each case it asserts -// BOTH: -// -// (a) the persisted string, re-parsed by go-sql-driver — the SAME parser that opens -// the connection — carries NO password (prove the secret is gone); and -// (b) every non-secret component (user, host:port, network, database, params, and the -// parseTime flag) survives redaction. -// -// A no-password DSN must additionally round-trip byte-for-byte unchanged. -func TestRedactPassword(t *testing.T) { - cases := []struct { - name string - dsn string - wantUser string - wantAddr string - wantNet string - wantDB string - wantParseTime bool - wantParams map[string]string - roundTripSame bool - }{ - { - name: "password present", - dsn: "bts:" + placeholderPW + "@tcp(127.0.0.1:3306)/beadsdb", - wantUser: "bts", wantAddr: "127.0.0.1:3306", wantNet: "tcp", wantDB: "beadsdb", - }, - { - // go-sql-driver splits the userinfo password on the LAST '@', so a password - // full of ':' '/' '#' '@' must still be stripped cleanly. - name: "password with special chars", - dsn: "bts:p@ss:w/rd#-" + placeholderPW + "@tcp(127.0.0.1:3306)/beadsdb", - wantUser: "bts", wantAddr: "127.0.0.1:3306", wantNet: "tcp", wantDB: "beadsdb", - }, - { - name: "password present, params + parseTime preserved", - dsn: "bts:" + placeholderPW + "@tcp(db.corp:3307)/beadsdb?parseTime=true&myapp=beads", - wantUser: "bts", wantAddr: "db.corp:3307", wantNet: "tcp", wantDB: "beadsdb", - wantParseTime: true, wantParams: map[string]string{"myapp": "beads"}, - }, - { - // The go-sql-driver grammar cannot carry a password without a username; - // FormatDSN drops the whole user:pass@ block, so redaction still removes the - // secret. (The loud refusal for this shape lives in withDatabase/WithPassword, - // not RedactPassword — here the invariant is only that the password is gone.) - name: "password but no username", - dsn: ":" + placeholderPW + "@tcp(127.0.0.1:3306)/beadsdb", - wantUser: "", wantAddr: "127.0.0.1:3306", wantNet: "tcp", wantDB: "beadsdb", - }, - { - // A canonical, param-free DSN must survive redaction byte-for-byte - // (go-sql-driver reorders query params on reformat, so the round-trip - // invariant is asserted only against a DSN already in canonical form). - name: "no password round-trips unchanged", - dsn: "bts@tcp(127.0.0.1:3306)/beadsdb", - wantUser: "bts", wantAddr: "127.0.0.1:3306", wantNet: "tcp", wantDB: "beadsdb", - roundTripSame: true, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got, err := RedactPassword(tc.dsn) - if err != nil { - t.Fatalf("RedactPassword(%q) errored: %v", tc.dsn, err) - } - - // (a) The authoritative security check: the same parser go-sql-driver uses to - // open the connection must find no password in the persisted string. - if pw := password(t, got); pw != "" { - t.Fatalf("PASSWORD LEAK: RedactPassword(%q) = %q still carries password %q", tc.dsn, got, pw) - } - - // (b) Non-secret components must survive. - cfg, err := gomysql.ParseDSN(got) - if err != nil { - t.Fatalf("redacted DSN no longer parses: %q: %v", got, err) - } - if cfg.User != tc.wantUser { - t.Errorf("user = %q, want %q", cfg.User, tc.wantUser) - } - if cfg.Addr != tc.wantAddr { - t.Errorf("addr = %q, want %q", cfg.Addr, tc.wantAddr) - } - if cfg.Net != tc.wantNet { - t.Errorf("net = %q, want %q", cfg.Net, tc.wantNet) - } - if cfg.DBName != tc.wantDB { - t.Errorf("dbname = %q, want %q", cfg.DBName, tc.wantDB) - } - if cfg.ParseTime != tc.wantParseTime { - t.Errorf("parseTime = %v, want %v (must be preserved)", cfg.ParseTime, tc.wantParseTime) - } - for k, want := range tc.wantParams { - if got := cfg.Params[k]; got != want { - t.Errorf("param %q = %q, want %q (must be preserved)", k, got, want) - } - } - - if tc.roundTripSame && got != tc.dsn { - t.Errorf("no-password DSN must round-trip unchanged:\n in = %q\n out = %q", tc.dsn, got) - } - }) - } -} diff --git a/internal/storage/mysql/schema.go b/internal/storage/mysql/schema.go deleted file mode 100644 index d178b02c9..000000000 --- a/internal/storage/mysql/schema.go +++ /dev/null @@ -1,162 +0,0 @@ -package mysql - -import ( - "context" - "database/sql" - _ "embed" - "errors" - "fmt" - "regexp" - "strings" -) - -// ddl is the MySQL backend schema, embedded from schema.sql. It is the Dolt -// `SHOW CREATE TABLE` output at HEAD (all migrations applied) with the handful of -// Dolt-vs-MySQL-8 fixes required by stock MySQL: TEXT/JSON literal defaults → -// expression form `DEFAULT (”)`, the overlong `spec_id` index → a 191-char prefix -// (InnoDB's 3072-byte key cap), and the one-target CHECK constraints dropped (MySQL 8 -// forbids a column in both a CHECK and an FK cascade; issueops enforces the invariant -// in code). No data-plane translation is needed — bd's canonical SQL is MySQL. -// -//go:embed schema.sql -var ddl string - -const ( - schemaVersion = "1" - schemaVersionKey = "mysql_schema_version" -) - -// identRe restricts a database name to a plain identifier so it can be safely -// interpolated into CREATE DATABASE (which cannot take a placeholder). -var identRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) - -// InitSchema creates the workspace database if absent (via a server connection with -// no default database), applies the DDL to it, seeds the default config rows on first -// provision, and stamps the schema version. baseDSN is the server DSN; database is the -// per-workspace database (MySQL's unit of isolation, analogous to a Postgres schema). -func InitSchema(ctx context.Context, baseDSN, database string) error { - if !identRe.MatchString(database) { - return fmt.Errorf("mysql: invalid database name %q", database) - } - serverDSN, err := withDatabase(baseDSN, "") - if err != nil { - return fmt.Errorf("mysql: parse DSN: %w", err) - } - srv, err := sql.Open("mysql", serverDSN) - if err != nil { - return fmt.Errorf("mysql: open server: %w", err) - } - if _, err := srv.ExecContext(ctx, "CREATE DATABASE IF NOT EXISTS `"+database+"` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin"); err != nil { - _ = srv.Close() - return fmt.Errorf("mysql: create database %q: %w", database, err) - } - _ = srv.Close() - - wsDSN, err := withDatabase(baseDSN, database) - if err != nil { - return fmt.Errorf("mysql: parse DSN: %w", err) - } - db, err := sql.Open("mysql", wsDSN) - if err != nil { - return fmt.Errorf("mysql: open workspace db: %w", err) - } - defer db.Close() - for _, stmt := range splitStatements(ddl) { - if _, err := db.ExecContext(ctx, stmt); err != nil { - return fmt.Errorf("mysql: exec DDL statement: %w\n-- statement:\n%s", err, stmt) - } - } - fresh, err := stampSchemaVersion(ctx, db) - if err != nil { - return err - } - if fresh { - if err := seedDefaultConfig(ctx, db); err != nil { - return err - } - } - return nil -} - -// defaultConfigSeeds mirrors migration 0016 exactly — the config rows a fresh Dolt -// workspace materializes on init. Seeded only on first provision (like the one-shot -// migration), so a `config unset` is never resurrected on the next open. -var defaultConfigSeeds = [][2]string{ - {"compaction_enabled", "false"}, - {"compact_tier1_days", "30"}, - {"compact_tier1_dep_levels", "2"}, - {"compact_tier2_days", "90"}, - {"compact_tier2_dep_levels", "5"}, - {"compact_tier2_commits", "100"}, - {"compact_batch_size", "50"}, - {"compact_parallel_workers", "5"}, - {"auto_compact_enabled", "false"}, -} - -func seedDefaultConfig(ctx context.Context, db *sql.DB) error { - for _, kv := range defaultConfigSeeds { - if _, err := db.ExecContext(ctx, - "INSERT IGNORE INTO config (`key`, `value`) VALUES (?, ?)", kv[0], kv[1]); err != nil { - return fmt.Errorf("mysql: seed default config %q: %w", kv[0], err) - } - } - return nil -} - -// stampSchemaVersion records schemaVersion in the metadata table on first provision -// and refuses to open a workspace written by a binary with a different schema version -// (the proof-wedge has no migrator). Returns fresh=true only on the first stamp. -func stampSchemaVersion(ctx context.Context, db *sql.DB) (fresh bool, err error) { - var stored string - err = db.QueryRowContext(ctx, "SELECT `value` FROM metadata WHERE `key` = ?", schemaVersionKey).Scan(&stored) - switch { - case errors.Is(err, sql.ErrNoRows): - // Provision runs on every open; concurrent first opens of the same - // workspace database can both reach here. INSERT IGNORE lets both - // succeed; only the opener that inserted the row (RowsAffected==1) is - // fresh and seeds defaults (seedDefaultConfig is already INSERT IGNORE). - res, ierr := db.ExecContext(ctx, "INSERT IGNORE INTO metadata (`key`, `value`) VALUES (?, ?)", schemaVersionKey, schemaVersion) - if ierr != nil { - return false, fmt.Errorf("mysql: stamp schema version: %w", ierr) - } - if n, _ := res.RowsAffected(); n == 1 { - return true, nil - } - // Lost the race: re-read what the winner stored and version-check it. - if rerr := db.QueryRowContext(ctx, "SELECT `value` FROM metadata WHERE `key` = ?", schemaVersionKey).Scan(&stored); rerr != nil { - return false, fmt.Errorf("mysql: read schema version after conflict: %w", rerr) - } - if stored != schemaVersion { - return false, fmt.Errorf("mysql: workspace schema version %s, this binary requires %s — no migrator in the proof-wedge, recreate the workspace or use a matching binary", stored, schemaVersion) - } - return false, nil - case err != nil: - return false, fmt.Errorf("mysql: read schema version: %w", err) - case stored != schemaVersion: - return false, fmt.Errorf("mysql: workspace schema version %s, this binary requires %s — no migrator in the proof-wedge, recreate the workspace or use a matching binary", stored, schemaVersion) - default: - return false, nil - } -} - -// splitStatements splits the DDL on ';' at statement boundaries. The MySQL schema has -// no procedural bodies (triggers/procedures) or embedded ';' — the CHECK constraints -// that would complicate this were removed — so a plain split suffices. Blank fragments -// (and SQL line comments) are dropped. -func splitStatements(ddl string) []string { - var stmts []string - for _, raw := range strings.Split(ddl, ";") { - var b strings.Builder - for _, line := range strings.Split(raw, "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "--") { - continue - } - b.WriteString(line) - b.WriteString("\n") - } - if s := strings.TrimSpace(b.String()); s != "" { - stmts = append(stmts, s) - } - } - return stmts -} diff --git a/internal/storage/mysql/schema.sql b/internal/storage/mysql/schema.sql deleted file mode 100644 index 4d6dc6537..000000000 --- a/internal/storage/mysql/schema.sql +++ /dev/null @@ -1,344 +0,0 @@ -CREATE TABLE IF NOT EXISTS `issues` ( - `id` varchar(255) NOT NULL, - `content_hash` varchar(64), - `title` varchar(500) NOT NULL, - `description` longtext NOT NULL, - `design` longtext NOT NULL, - `acceptance_criteria` longtext NOT NULL, - `notes` longtext NOT NULL, - `status` varchar(32) NOT NULL DEFAULT 'open', - `priority` int NOT NULL DEFAULT '2', - `issue_type` varchar(32) NOT NULL DEFAULT 'task', - `assignee` varchar(255), - `estimated_minutes` int, - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `created_by` varchar(255) DEFAULT '', - `owner` varchar(255) DEFAULT '', - `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `closed_at` datetime, - `closed_by_session` varchar(255) DEFAULT '', - `external_ref` varchar(255), - `spec_id` varchar(1024), - `compaction_level` int DEFAULT '0', - `compacted_at` datetime, - `compacted_at_commit` varchar(64), - `original_size` int, - `sender` varchar(255) DEFAULT '', - `ephemeral` tinyint(1) DEFAULT '0', - `wisp_type` varchar(32) DEFAULT '', - `pinned` tinyint(1) DEFAULT '0', - `is_template` tinyint(1) DEFAULT '0', - `mol_type` varchar(32) DEFAULT '', - `work_type` varchar(32) DEFAULT 'mutex', - `source_system` varchar(255) DEFAULT '', - `metadata` json DEFAULT (json_object()), - `source_repo` varchar(512) DEFAULT '', - `close_reason` longtext DEFAULT (''), - `event_kind` varchar(32) DEFAULT '', - `actor` varchar(255) DEFAULT '', - `target` varchar(255) DEFAULT '', - `payload` text DEFAULT (''), - `await_type` varchar(32) DEFAULT '', - `await_id` varchar(255) DEFAULT '', - `timeout_ns` bigint DEFAULT '0', - `waiters` text DEFAULT (''), - `hook_bead` varchar(255) DEFAULT '', - `role_bead` varchar(255) DEFAULT '', - `agent_state` varchar(32) DEFAULT '', - `last_activity` datetime, - `role_type` varchar(32) DEFAULT '', - `rig` varchar(255) DEFAULT '', - `due_at` datetime, - `defer_until` datetime, - `no_history` tinyint(1) DEFAULT '0', - `started_at` datetime, - `is_blocked` tinyint(1) NOT NULL DEFAULT '0', - `row_lock` bigint NOT NULL DEFAULT 0, - PRIMARY KEY (`id`), - KEY `idx_issues_assignee` (`assignee`), - KEY `idx_issues_created_at` (`created_at`), - KEY `idx_issues_defer_until` (`defer_until`), - KEY `idx_issues_external_ref` (`external_ref`), - KEY `idx_issues_is_blocked` (`is_blocked`,`status`), - KEY `idx_issues_issue_type` (`issue_type`), - KEY `idx_issues_priority` (`priority`), - KEY `idx_issues_spec_id` (`spec_id`(191)), - KEY `idx_issues_status_updated_at` (`status`,`updated_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - --- Ephemeral claim leases (bd-lrgn1): node-local, never exported by backup. --- One row per live claim. See issueops.UpsertLeaseInTx for the invariant. -CREATE TABLE IF NOT EXISTS `leases` ( - `issue_id` varchar(255) NOT NULL, - `holder` varchar(255) NOT NULL, - `granted_at` datetime NOT NULL, - `lease_expires_at` datetime NOT NULL, - `heartbeat_at` datetime NOT NULL, - PRIMARY KEY (`issue_id`), - KEY `idx_leases_expires` (`lease_expires_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `wisps` ( - `id` varchar(255) NOT NULL, - `content_hash` varchar(64), - `title` varchar(500) NOT NULL, - `description` longtext NOT NULL DEFAULT (''), - `design` longtext NOT NULL DEFAULT (''), - `acceptance_criteria` longtext NOT NULL DEFAULT (''), - `notes` longtext NOT NULL DEFAULT (''), - `status` varchar(32) NOT NULL DEFAULT 'open', - `priority` int NOT NULL DEFAULT '2', - `issue_type` varchar(32) NOT NULL DEFAULT 'task', - `assignee` varchar(255), - `estimated_minutes` int, - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `created_by` varchar(255) DEFAULT '', - `owner` varchar(255) DEFAULT '', - `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `closed_at` datetime, - `closed_by_session` varchar(255) DEFAULT '', - `external_ref` varchar(255), - `spec_id` varchar(1024), - `compaction_level` int DEFAULT '0', - `compacted_at` datetime, - `compacted_at_commit` varchar(64), - `original_size` int, - `sender` varchar(255) DEFAULT '', - `ephemeral` tinyint(1) DEFAULT '0', - `wisp_type` varchar(32) DEFAULT '', - `pinned` tinyint(1) DEFAULT '0', - `is_template` tinyint(1) DEFAULT '0', - `mol_type` varchar(32) DEFAULT '', - `work_type` varchar(32) DEFAULT 'mutex', - `source_system` varchar(255) DEFAULT '', - `metadata` json DEFAULT (json_object()), - `source_repo` varchar(512) DEFAULT '', - `close_reason` longtext DEFAULT (''), - `event_kind` varchar(32) DEFAULT '', - `actor` varchar(255) DEFAULT '', - `target` varchar(255) DEFAULT '', - `payload` text DEFAULT (''), - `await_type` varchar(32) DEFAULT '', - `await_id` varchar(255) DEFAULT '', - `timeout_ns` bigint DEFAULT '0', - `waiters` text DEFAULT (''), - `hook_bead` varchar(255) DEFAULT '', - `role_bead` varchar(255) DEFAULT '', - `agent_state` varchar(32) DEFAULT '', - `last_activity` datetime, - `role_type` varchar(32) DEFAULT '', - `rig` varchar(255) DEFAULT '', - `due_at` datetime, - `defer_until` datetime, - `no_history` tinyint(1) DEFAULT '0', - `started_at` datetime, - `is_blocked` tinyint(1) NOT NULL DEFAULT '0', - `row_lock` bigint NOT NULL DEFAULT 0, - PRIMARY KEY (`id`), - KEY `idx_wisps_assignee` (`assignee`), - KEY `idx_wisps_created_at` (`created_at`), - KEY `idx_wisps_external_ref` (`external_ref`), - KEY `idx_wisps_is_blocked` (`is_blocked`,`status`), - KEY `idx_wisps_issue_type` (`issue_type`), - KEY `idx_wisps_priority` (`priority`), - KEY `idx_wisps_spec_id` (`spec_id`(191)), - KEY `idx_wisps_status` (`status`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `labels` ( - `issue_id` varchar(255) NOT NULL, - `label` varchar(255) NOT NULL, - PRIMARY KEY (`issue_id`,`label`), - KEY `idx_labels_label` (`label`), - CONSTRAINT `fk_labels_issue` FOREIGN KEY (`issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `wisp_labels` ( - `issue_id` varchar(255) NOT NULL, - `label` varchar(255) NOT NULL, - PRIMARY KEY (`issue_id`,`label`), - KEY `idx_wisp_labels_label` (`label`), - CONSTRAINT `fk_wisp_labels_issue` FOREIGN KEY (`issue_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `dependencies` ( - `id` char(36) NOT NULL, - `issue_id` varchar(255) NOT NULL, - `type` varchar(32) NOT NULL DEFAULT 'blocks', - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `created_by` varchar(255) NOT NULL, - `metadata` json DEFAULT (json_object()), - `thread_id` varchar(255) DEFAULT '', - `depends_on_issue_id` varchar(255), - `depends_on_wisp_id` varchar(255), - `depends_on_external` varchar(255), - PRIMARY KEY (`id`), - KEY `idx_dep_external_target` (`depends_on_external`), - KEY `idx_dep_issue_target` (`depends_on_issue_id`), - KEY `idx_dep_type_external` (`type`,`depends_on_external`), - KEY `idx_dep_type_issue` (`type`,`depends_on_issue_id`), - KEY `idx_dep_type_wisp` (`type`,`depends_on_wisp_id`), - KEY `idx_dep_wisp_target` (`depends_on_wisp_id`), - KEY `idx_dependencies_issue` (`issue_id`), - KEY `idx_dependencies_thread` (`thread_id`), - UNIQUE KEY `uk_dep_external_target` (`issue_id`,`depends_on_external`), - UNIQUE KEY `uk_dep_issue_target` (`issue_id`,`depends_on_issue_id`), - UNIQUE KEY `uk_dep_wisp_target` (`issue_id`,`depends_on_wisp_id`), - CONSTRAINT `fk_dep_issue` FOREIGN KEY (`issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT `fk_dep_issue_target` FOREIGN KEY (`depends_on_issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `wisp_dependencies` ( - `id` char(36) NOT NULL, - `issue_id` varchar(255) NOT NULL, - `depends_on_issue_id` varchar(255), - `depends_on_wisp_id` varchar(255), - `depends_on_external` varchar(255), - `type` varchar(32) NOT NULL DEFAULT 'blocks', - `created_at` datetime DEFAULT CURRENT_TIMESTAMP, - `created_by` varchar(255) DEFAULT '', - `metadata` json DEFAULT (json_object()), - `thread_id` varchar(255) DEFAULT '', - PRIMARY KEY (`id`), - KEY `fk_wisp_dep_issue_target` (`depends_on_issue_id`), - KEY `fk_wisp_dep_wisp_target` (`depends_on_wisp_id`), - KEY `idx_wisp_dep_type` (`type`), - KEY `idx_wisp_dep_type_external` (`type`,`depends_on_external`), - KEY `idx_wisp_dep_type_issue` (`type`,`depends_on_issue_id`), - KEY `idx_wisp_dep_type_wisp` (`type`,`depends_on_wisp_id`), - UNIQUE KEY `uk_wisp_dep_external_target` (`issue_id`,`depends_on_external`), - UNIQUE KEY `uk_wisp_dep_issue_target` (`issue_id`,`depends_on_issue_id`), - UNIQUE KEY `uk_wisp_dep_wisp_target` (`issue_id`,`depends_on_wisp_id`), - CONSTRAINT `fk_wisp_dep_issue` FOREIGN KEY (`issue_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT `fk_wisp_dep_issue_target` FOREIGN KEY (`depends_on_issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT `fk_wisp_dep_wisp_target` FOREIGN KEY (`depends_on_wisp_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `comments` ( - `id` char(36) NOT NULL, - `issue_id` varchar(255) NOT NULL, - `author` varchar(255) NOT NULL, - `text` longtext NOT NULL, - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - KEY `idx_comments_created_at` (`created_at`), - KEY `idx_comments_issue` (`issue_id`), - CONSTRAINT `fk_comments_issue` FOREIGN KEY (`issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `wisp_comments` ( - `id` char(36) NOT NULL, - `issue_id` varchar(255) NOT NULL, - `author` varchar(255) DEFAULT '', - `text` text NOT NULL, - `created_at` datetime DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - KEY `idx_wisp_comments_issue` (`issue_id`), - CONSTRAINT `fk_wisp_comments_issue` FOREIGN KEY (`issue_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `events` ( - `id` char(36) NOT NULL, - `issue_id` varchar(255) NOT NULL, - `event_type` varchar(32) NOT NULL, - `actor` varchar(255) NOT NULL, - `old_value` longtext, - `new_value` longtext, - `comment` text, - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - KEY `idx_events_created_at` (`created_at`), - KEY `idx_events_issue` (`issue_id`), - CONSTRAINT `fk_events_issue` FOREIGN KEY (`issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `wisp_events` ( - `id` char(36) NOT NULL, - `issue_id` varchar(255) NOT NULL, - `event_type` varchar(32) NOT NULL, - `actor` varchar(255) DEFAULT '', - `old_value` longtext DEFAULT (''), - `new_value` longtext DEFAULT (''), - `comment` text DEFAULT (''), - `created_at` datetime DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - KEY `idx_wisp_events_created_at` (`created_at`), - KEY `idx_wisp_events_issue` (`issue_id`), - CONSTRAINT `fk_wisp_events_issue` FOREIGN KEY (`issue_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `config` ( - `key` varchar(255) NOT NULL, - `value` text NOT NULL, - PRIMARY KEY (`key`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `metadata` ( - `key` varchar(255) NOT NULL, - `value` text NOT NULL, - PRIMARY KEY (`key`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `local_metadata` ( - `key` varchar(255) NOT NULL, - `value` text NOT NULL DEFAULT (''), - PRIMARY KEY (`key`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `issue_counter` ( - `prefix` varchar(255) NOT NULL, - `last_id` int NOT NULL DEFAULT '0', - PRIMARY KEY (`prefix`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `child_counters` ( - `parent_id` varchar(255) NOT NULL, - `last_child` int NOT NULL DEFAULT '0', - PRIMARY KEY (`parent_id`), - CONSTRAINT `fk_counter_parent` FOREIGN KEY (`parent_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `wisp_child_counters` ( - `parent_id` varchar(255) NOT NULL, - `last_child` int NOT NULL DEFAULT '0', - PRIMARY KEY (`parent_id`), - CONSTRAINT `fk_wisp_child_counters_parent` FOREIGN KEY (`parent_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `custom_statuses` ( - `name` varchar(64) NOT NULL, - `category` varchar(32) NOT NULL DEFAULT 'unspecified', - PRIMARY KEY (`name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `custom_types` ( - `name` varchar(64) NOT NULL, - PRIMARY KEY (`name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; - - -CREATE TABLE IF NOT EXISTS `repo_mtimes` ( - `repo_path` varchar(512) NOT NULL, - `jsonl_path` varchar(512) NOT NULL, - `mtime_ns` bigint NOT NULL, - `last_checked` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`repo_path`), - KEY `idx_repo_mtimes_checked` (`last_checked`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; diff --git a/internal/storage/mysql/seed_test.go b/internal/storage/mysql/seed_test.go deleted file mode 100644 index 6a6f8353b..000000000 --- a/internal/storage/mysql/seed_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package mysql - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" -) - -// TestSeedOnlyOnFirstProvision guards the same regression the Postgres backend has: -// InitSchema runs on every open, so seeding default config unconditionally would -// resurrect a key the user `config unset`. Seeding must be gated on first provision. -func TestSeedOnlyOnFirstProvision(t *testing.T) { - url := os.Getenv("BEADS_MYSQL_TEST_URL") - if url == "" { - t.Skip("BEADS_MYSQL_TEST_URL not set") - } - database := fmt.Sprintf("seedonce_%d", time.Now().UnixNano()) - ctx := context.Background() - t.Cleanup(func() { - if serverDSN, e := withDatabase(url, ""); e == nil { - if srv, e2 := sql.Open("mysql", serverDSN); e2 == nil { - _, _ = srv.ExecContext(context.Background(), "DROP DATABASE IF EXISTS `"+database+"`") - _ = srv.Close() - } - } - }) - - st, err := Provision(ctx, url, database) - if err != nil { - t.Fatalf("Provision: %v", err) - } - if v, _ := st.GetConfig(ctx, "auto_compact_enabled"); v != "false" { - t.Fatalf("first provision: want seeded auto_compact_enabled=false, got %q", v) - } - if err := st.DeleteConfig(ctx, "auto_compact_enabled"); err != nil { - t.Fatalf("DeleteConfig: %v", err) - } - if v, _ := st.GetConfig(ctx, "auto_compact_enabled"); v != "" { - t.Fatalf("after unset: want empty, got %q", v) - } - _ = st.Close() - - st2, err := Provision(ctx, url, database) - if err != nil { - t.Fatalf("re-Provision: %v", err) - } - defer st2.Close() - if v, _ := st2.GetConfig(ctx, "auto_compact_enabled"); v != "" { - t.Fatalf("REGRESSION: re-provision resurrected an unset key: auto_compact_enabled=%q (want empty)", v) - } -} diff --git a/internal/storage/mysql/store.go b/internal/storage/mysql/store.go deleted file mode 100644 index 14cd7d325..000000000 --- a/internal/storage/mysql/store.go +++ /dev/null @@ -1,69 +0,0 @@ -// Package mysql is the MySQL backend for bd: the shared sqlkit.Store bundled with -// the identity MySQL dialect and a no-op readiness strategy, plus the non-Dolt -// markers. bd's canonical SQL is MySQL-dialect, so unlike Postgres there is no -// translating driver — the only backend-specific work is a curated MySQL-8 schema -// (Dolt is more permissive than stock MySQL 8 at the DDL level). -package mysql - -import ( - "context" - - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/storage/sqlkit" -) - -// Backend is the metadata.json backend identifier for this store. -const Backend = "mysql" - -// Store is the MySQL backend. It embeds *sqlkit.Store (inheriting every core method) -// and the generated typed-unsupported shell for the non-core surface. -type Store struct { - *sqlkit.Store - unsupportedDoltStorage -} - -// Compile-time guard on the generated skip list: the composite backend (real -// *sqlkit.Store methods plus the typed-unsupported shell) must fully satisfy -// storage.DoltStorage. This fails the build on skip-list drift — a skipped -// method sqlkit does not actually implement (missing method), or a method both -// implemented and stubbed (ambiguous selector) — so no method can silently -// become "unsupported". -var _ storage.DoltStorage = (*Store)(nil) - -// Config configures a MySQL backend. DSN targets the per-workspace database; the -// password is merged from env at open and never persisted to metadata.json. -type Config struct { - DSN string -} - -// New builds the MySQL backend from its strategy bundle: the identity mysql dialect -// and a no-op readiness strategy (issueops already maintains is_blocked in-tx, exactly -// like the Dolt reference). -func New(ctx context.Context, cfg Config) (*Store, error) { - base, err := sqlkit.New(ctx, sqlkit.Config{ - Dialect: mysqlDialect{dsn: cfg.DSN}, - Readiness: sqlkit.NoopReadiness{}, - }) - if err != nil { - return nil, err - } - return &Store{Store: base}, nil -} - -// CommitGraphUnsupported tells cmd/bd's PostRun to skip Dolt-only maintenance. -func (s *Store) CommitGraphUnsupported() bool { return true } - -// CommitPending is a no-op on a non-version-controlled backend. -func (s *Store) CommitPending(_ context.Context, _ string) (bool, error) { return false, nil } - -// Commit and its variants are no-ops on MySQL, not unsupported: every sqlkit write -// already committed its own SQL transaction, so the data is durable. Command paths -// call store.Commit() opportunistically; returning nil lets those writes stand. -func (s *Store) Commit(_ context.Context, _ string) error { return nil } -func (s *Store) CommitWithConfig(_ context.Context, _ string) error { return nil } -func (s *Store) CommitMergeResolution(_ context.Context, _ string) error { return nil } - -var ( - _ storage.DoltStorage = (*Store)(nil) - _ storage.NonCommitGraphBackend = (*Store)(nil) -) diff --git a/internal/storage/mysql/unsupported.go b/internal/storage/mysql/unsupported.go deleted file mode 100644 index 6fe9487c0..000000000 --- a/internal/storage/mysql/unsupported.go +++ /dev/null @@ -1,17 +0,0 @@ -package mysql - -import ( - "github.com/steveyegge/beads/internal/storage" -) - -// Regeneration mirrors postgres: the mysql shell is the exact complement of -// *sqlkit.Store's method set (identical base + identical Commit/CommitGraph overrides, -// so the complement is identical to postgres's shell modulo package name). -// -//go:generate go run ../unsupportedgen -pkg mysql -src .. -out unsupported_gen.go -type DoltStorage -skip AddDependency,AddIssueComment,AddLabel,ClaimIssue,ClaimReadyIssue,Close,CloseIssue,Commit,CommitMergeResolution,CommitPending,CommitWithConfig,CountDependencies,CountDependents,CountEvents,CountIssueComments,CountIssues,CountIssuesByGroup,CreateIssue,CreateIssues,DeleteConfig,DeleteIssue,DeleteIssues,DetectCycles,GetAllConfig,GetBlockedIssues,GetBlockingInfoForIssues,GetCommentCounts,GetCommentsForIssues,GetConfig,GetCustomStatuses,GetCustomStatusesDetailed,GetCustomTypes,GetDependencies,GetDependenciesWithMetadata,GetDependencyCounts,GetDependencyRecords,GetDependencyRecordsForIssues,GetDependencyTree,GetDependents,GetDependentsWithMetadata,GetEpicsEligibleForClosure,GetEvents,GetInfraTypes,GetIssue,GetIssueByExternalRef,GetIssueComments,GetIssuesByIDs,GetIssuesByLabel,GetLabels,GetLabelsForIssues,GetLocalMetadata,GetMetadata,GetNewlyUnblockedByClose,GetNextChildID,GetReadyWork,GetReadyWorkWithCounts,GetStaleIssues,GetStatistics,IsBlocked,IsInfraTypeCtx,IterBlockedIssues,IterDependenciesWithMetadata,IterDependentsWithMetadata,IterEvents,IterIssueComments,IterIssues,IterReadyWork,IterWisps,ListWisps,RemoveDependency,RemoveLabel,ReopenIssue,RunInTransaction,SearchIssues,SearchIssuesWithCounts,SetConfig,SetLocalMetadata,SetMetadata,UpdateIssue,UpdateIssueType,AddComment,ClearRepoMtime,CountDependentsByStatus,DeleteIssuesBySourceRepo,FindWispDependentsRecursive,GetAllDependencyRecords,GetAllEventsSince,GetMoleculeLastActivity,GetMoleculeProgress,GetRepoMtime,ImportIssueComment,IterAllDependencyRecords,IterAllEventsSince,PromoteFromEphemeral,SetRepoMtime,UpdateIssueID,CreateIssuesWithFullOptions,SlotClear,SlotGet,SlotSet,SearchIssueIDs,HeartbeatIssue,ReclaimExpiredLeases,UnclaimIssue - -// errUnsupported is the constructor every generated stub in unsupported_gen.go calls. -// It returns the backend-agnostic *storage.ErrUnsupported sentinel; Backend is "mysql". -func errUnsupported(op string) error { - return &storage.ErrUnsupported{Op: op, Backend: Backend} -} diff --git a/internal/storage/mysql/unsupported_gen.go b/internal/storage/mysql/unsupported_gen.go deleted file mode 100644 index 823cc4b8e..000000000 --- a/internal/storage/mysql/unsupported_gen.go +++ /dev/null @@ -1,236 +0,0 @@ -// Code generated by gen (mysql typed-unsupported shell); DO NOT EDIT. - -package mysql - -import ( - "context" - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/types" -) - -// unsupportedDoltStorage is the generated typed-unsupported shell for storage.DoltStorage. -// Every method returns *storage.ErrUnsupported; embed it and override the -// real slice. DO NOT hand-edit — regenerate with `go generate ./...`. -type unsupportedDoltStorage struct{} - -func (unsupportedDoltStorage) MergeSlotCreate(_ context.Context, _ string) (_ *types.Issue, err error) { - err = errUnsupported("MergeSlotCreate") - return -} - -func (unsupportedDoltStorage) MergeSlotCheck(_ context.Context) (_ *storage.MergeSlotStatus, err error) { - err = errUnsupported("MergeSlotCheck") - return -} - -func (unsupportedDoltStorage) MergeSlotAcquire(_ context.Context, _ string, _ string, _ bool) (_ *storage.MergeSlotResult, err error) { - err = errUnsupported("MergeSlotAcquire") - return -} - -func (unsupportedDoltStorage) MergeSlotRelease(_ context.Context, _ string, _ string) (err error) { - err = errUnsupported("MergeSlotRelease") - return -} - -func (unsupportedDoltStorage) Branch(_ context.Context, _ string) (err error) { - err = errUnsupported("Branch") - return -} - -func (unsupportedDoltStorage) Checkout(_ context.Context, _ string) (err error) { - err = errUnsupported("Checkout") - return -} - -func (unsupportedDoltStorage) CurrentBranch(_ context.Context) (_ string, err error) { - err = errUnsupported("CurrentBranch") - return -} - -func (unsupportedDoltStorage) DeleteBranch(_ context.Context, _ string) (err error) { - err = errUnsupported("DeleteBranch") - return -} - -func (unsupportedDoltStorage) ListBranches(_ context.Context) (_ []string, err error) { - err = errUnsupported("ListBranches") - return -} - -func (unsupportedDoltStorage) CommitExists(_ context.Context, _ string) (_ bool, err error) { - err = errUnsupported("CommitExists") - return -} - -func (unsupportedDoltStorage) GetCurrentCommit(_ context.Context) (_ string, err error) { - err = errUnsupported("GetCurrentCommit") - return -} - -func (unsupportedDoltStorage) Status(_ context.Context) (_ *storage.Status, err error) { - err = errUnsupported("Status") - return -} - -func (unsupportedDoltStorage) Log(_ context.Context, _ int) (_ []storage.CommitInfo, err error) { - err = errUnsupported("Log") - return -} - -func (unsupportedDoltStorage) Merge(_ context.Context, _ string) (_ []storage.Conflict, err error) { - err = errUnsupported("Merge") - return -} - -func (unsupportedDoltStorage) GetConflicts(_ context.Context) (_ []storage.Conflict, err error) { - err = errUnsupported("GetConflicts") - return -} - -func (unsupportedDoltStorage) ResolveConflicts(_ context.Context, _ string, _ string) (err error) { - err = errUnsupported("ResolveConflicts") - return -} - -func (unsupportedDoltStorage) History(_ context.Context, _ string) (_ []*storage.HistoryEntry, err error) { - err = errUnsupported("History") - return -} - -func (unsupportedDoltStorage) AsOf(_ context.Context, _ string, _ string) (_ *types.Issue, err error) { - err = errUnsupported("AsOf") - return -} - -func (unsupportedDoltStorage) Diff(_ context.Context, _ string, _ string) (_ []*storage.DiffEntry, err error) { - err = errUnsupported("Diff") - return -} - -func (unsupportedDoltStorage) AddRemote(_ context.Context, _ string, _ string) (err error) { - err = errUnsupported("AddRemote") - return -} - -func (unsupportedDoltStorage) RemoveRemote(_ context.Context, _ string) (err error) { - err = errUnsupported("RemoveRemote") - return -} - -func (unsupportedDoltStorage) HasRemote(_ context.Context, _ string) (_ bool, err error) { - err = errUnsupported("HasRemote") - return -} - -func (unsupportedDoltStorage) ListRemotes(_ context.Context) (_ []storage.RemoteInfo, err error) { - err = errUnsupported("ListRemotes") - return -} - -func (unsupportedDoltStorage) Push(_ context.Context) (err error) { - err = errUnsupported("Push") - return -} - -func (unsupportedDoltStorage) Pull(_ context.Context) (err error) { - err = errUnsupported("Pull") - return -} - -func (unsupportedDoltStorage) ForcePush(_ context.Context) (err error) { - err = errUnsupported("ForcePush") - return -} - -func (unsupportedDoltStorage) PushRemote(_ context.Context, _ string, _ bool) (err error) { - err = errUnsupported("PushRemote") - return -} - -func (unsupportedDoltStorage) PullRemote(_ context.Context, _ string) (err error) { - err = errUnsupported("PullRemote") - return -} - -func (unsupportedDoltStorage) Fetch(_ context.Context, _ string) (err error) { - err = errUnsupported("Fetch") - return -} - -func (unsupportedDoltStorage) PushTo(_ context.Context, _ string) (err error) { - err = errUnsupported("PushTo") - return -} - -func (unsupportedDoltStorage) PullFrom(_ context.Context, _ string) (_ []storage.Conflict, err error) { - err = errUnsupported("PullFrom") - return -} - -func (unsupportedDoltStorage) Sync(_ context.Context, _ string, _ string) (_ *storage.SyncResult, err error) { - err = errUnsupported("Sync") - return -} - -func (unsupportedDoltStorage) SyncStatus(_ context.Context, _ string) (_ *storage.SyncStatus, err error) { - err = errUnsupported("SyncStatus") - return -} - -func (unsupportedDoltStorage) AddFederationPeer(_ context.Context, _ *storage.FederationPeer) (err error) { - err = errUnsupported("AddFederationPeer") - return -} - -func (unsupportedDoltStorage) GetFederationPeer(_ context.Context, _ string) (_ *storage.FederationPeer, err error) { - err = errUnsupported("GetFederationPeer") - return -} - -func (unsupportedDoltStorage) ListFederationPeers(_ context.Context) (_ []*storage.FederationPeer, err error) { - err = errUnsupported("ListFederationPeers") - return -} - -func (unsupportedDoltStorage) RemoveFederationPeer(_ context.Context, _ string) (err error) { - err = errUnsupported("RemoveFederationPeer") - return -} - -func (unsupportedDoltStorage) CheckEligibility(_ context.Context, _ string, _ int) (_ bool, _ string, err error) { - err = errUnsupported("CheckEligibility") - return -} - -func (unsupportedDoltStorage) ApplyCompaction(_ context.Context, _ string, _ int, _ int, _ int, _ string) (err error) { - err = errUnsupported("ApplyCompaction") - return -} - -func (unsupportedDoltStorage) GetTier1Candidates(_ context.Context) (_ []*types.CompactionCandidate, err error) { - err = errUnsupported("GetTier1Candidates") - return -} - -func (unsupportedDoltStorage) GetTier2Candidates(_ context.Context) (_ []*types.CompactionCandidate, err error) { - err = errUnsupported("GetTier2Candidates") - return -} - -func (unsupportedDoltStorage) SnapshotIssue(_ context.Context, _ string, _ int) (err error) { - err = errUnsupported("SnapshotIssue") - return -} - -func (unsupportedDoltStorage) GetCompactionSnapshot(_ context.Context, _ string) (_ *types.IssueSnapshot, err error) { - err = errUnsupported("GetCompactionSnapshot") - return -} - -func (unsupportedDoltStorage) RestoreFromSnapshot(_ context.Context, _ string) (_ *types.IssueSnapshot, err error) { - err = errUnsupported("RestoreFromSnapshot") - return -} - -// NOTE: partial shell (104 methods skipped); the embedding composite must assert storage.DoltStorage itself. diff --git a/internal/storage/mysqldialect/credential.go b/internal/storage/mysqldialect/credential.go deleted file mode 100644 index 1a034233b..000000000 --- a/internal/storage/mysqldialect/credential.go +++ /dev/null @@ -1,83 +0,0 @@ -package mysqldialect - -import ( - "fmt" - "net" - "strconv" - - "github.com/go-sql-driver/mysql" -) - -// HasPassword reports whether the DSN already carries a password. A DSN that already -// has one (a full BEADS_MYSQL_URL, or an explicit --mysql-url) wins outright: the -// credential ladder is not applied on top of it. -func HasPassword(dsn string) bool { - cfg, err := mysql.ParseDSN(dsn) - return err == nil && cfg.Passwd != "" -} - -// WithCredential returns dsn with the resolved credential placed: the password -// always, and the username only when non-empty (a Vault-style dynamic user/password -// pair) — an empty username leaves the DSN's user untouched. It uses go-sql-driver's -// own parser (never string surgery) and VERIFIES the round-trip. -// -// The grammar has one sharp edge pgdialect does not: it cannot carry a password -// without a username (FormatDSN emits the user:pass@ block only when a user is -// present), so a userless result is refused. But a provided username satisfies that -// requirement, making a bare tcp(host)/ base DSN plus a dynamic user/password pair a -// valid configuration. This is the counterpart to mysql.RedactPassword. -func WithCredential(dsn, username, password string) (string, error) { - cfg, err := mysql.ParseDSN(dsn) - if err != nil { - return "", fmt.Errorf("cannot place credential into the connection string (unparseable DSN): %w", err) - } - if username != "" { - cfg.User = username - } - if cfg.User == "" { - return "", fmt.Errorf("cannot place a password: the DSN has no username, and the MySQL DSN grammar cannot carry a password without one; add a user to the DSN") - } - cfg.Passwd = password - out := cfg.FormatDSN() - - again, err := mysql.ParseDSN(out) - if err != nil { - return "", fmt.Errorf("cannot place credential into the connection string (unparseable result): %w", err) - } - if again.Passwd != password { - return "", fmt.Errorf("password did not round-trip into the connection string; refusing to connect with a wrong or empty password") - } - if username != "" && again.User != username { - return "", fmt.Errorf("username did not round-trip into the connection string; refusing to connect as the wrong user") - } - return out, nil -} - -// WithPassword places only the password, preserving the DSN's user. It is -// WithCredential with no username override. -func WithPassword(dsn, password string) (string, error) { - return WithCredential(dsn, "", password) -} - -// HostPort returns the TCP endpoint go-sql-driver would dial for dsn, for keying the -// [host:port] credentials-file lookup. ok=false for non-TCP networks (a unix socket -// has no port) or an unparseable DSN; the caller degrades the file rung to -// not-configured rather than failing. -func HostPort(dsn string) (host string, port int, ok bool) { - cfg, err := mysql.ParseDSN(dsn) - if err != nil { - return "", 0, false - } - if cfg.Net != "tcp" { - return "", 0, false - } - h, p, err := net.SplitHostPort(cfg.Addr) - if err != nil { - return "", 0, false - } - port, err = strconv.Atoi(p) - if err != nil { - return "", 0, false - } - return h, port, true -} diff --git a/internal/storage/mysqldialect/credential_extra_test.go b/internal/storage/mysqldialect/credential_extra_test.go deleted file mode 100644 index f43019e24..000000000 --- a/internal/storage/mysqldialect/credential_extra_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package mysqldialect - -import ( - "testing" - - "github.com/go-sql-driver/mysql" -) - -func mustUser(t *testing.T, out string) string { - t.Helper() - cfg, err := mysql.ParseDSN(out) - if err != nil { - t.Fatalf("mysql.ParseDSN(%q): %v", out, err) - } - return cfg.User -} - -func TestWithCredentialOverridesUser(t *testing.T) { - out, err := WithCredential("old@tcp(h:3306)/db", "vu", "pw") - if err != nil { - t.Fatal(err) - } - if u := mustUser(t, out); u != "vu" { - t.Fatalf("user = %q, want vu", u) - } - if pw := mustPassword(t, out); pw != "pw" { - t.Fatalf("password = %q, want pw", pw) - } -} - -func TestWithCredentialEmptyUsernameKeepsUser(t *testing.T) { - out, err := WithCredential("old@tcp(h:3306)/db", "", "pw") - if err != nil { - t.Fatal(err) - } - if u := mustUser(t, out); u != "old" { - t.Fatalf("user = %q, want old (preserved)", u) - } -} - -// A provided username satisfies the grammar's user requirement, making a userless base -// DSN plus a dynamic user/password pair valid. -func TestWithCredentialUsernameSatisfiesUserlessDSN(t *testing.T) { - out, err := WithCredential("tcp(h:3306)/db", "vu", "pw") - if err != nil { - t.Fatal(err) - } - if u := mustUser(t, out); u != "vu" { - t.Fatalf("user = %q, want vu", u) - } - if pw := mustPassword(t, out); pw != "pw" { - t.Fatalf("password = %q, want pw", pw) - } -} - -// Password-only against a userless DSN still fails (the guard is intact). -func TestWithCredentialUserlessStillFails(t *testing.T) { - if _, err := WithCredential("tcp(h:3306)/db", "", "pw"); err == nil { - t.Fatal("expected error: a userless DSN cannot carry a password without a username") - } -} - -func TestHostPort(t *testing.T) { - cases := []struct { - dsn string - host string - port int - ok bool - }{ - {"u@tcp(10.0.0.5:55441)/", "10.0.0.5", 55441, true}, - {"u@/", "127.0.0.1", 3306, true}, // tcp Addr defaults to 127.0.0.1:3306 - {"u@tcp(dbhost)/", "dbhost", 3306, true}, // ensureHavePort appends :3306 - {"u@unix(/tmp/mysql.sock)/", "", 0, false}, // non-tcp -> not ok - } - for _, c := range cases { - h, p, ok := HostPort(c.dsn) - if ok != c.ok || (ok && (h != c.host || p != c.port)) { - t.Errorf("HostPort(%q) = (%q,%d,%v), want (%q,%d,%v)", c.dsn, h, p, ok, c.host, c.port, c.ok) - } - } -} diff --git a/internal/storage/mysqldialect/credential_test.go b/internal/storage/mysqldialect/credential_test.go deleted file mode 100644 index 49ef6affd..000000000 --- a/internal/storage/mysqldialect/credential_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package mysqldialect - -import ( - "testing" - - "github.com/go-sql-driver/mysql" -) - -func TestHasPassword(t *testing.T) { - cases := []struct { - dsn string - want bool - }{ - {"bts:secret@tcp(127.0.0.1:3306)/db", true}, - {"bts@tcp(127.0.0.1:3306)/db", false}, - {"bts@/", false}, // no-tcp form parses, no password - {"", false}, // empty parses to a default config - {"bts:pw@tcp(127.0.0.1:3306)", false}, // missing trailing slash -> parse error - } - for _, c := range cases { - if got := HasPassword(c.dsn); got != c.want { - t.Errorf("HasPassword(%q) = %v, want %v", c.dsn, got, c.want) - } - } -} - -// mustPassword parses out and returns the password go-sql-driver sees. -func mustPassword(t *testing.T, out string) string { - t.Helper() - cfg, err := mysql.ParseDSN(out) - if err != nil { - t.Fatalf("mysql.ParseDSN(%q): %v", out, err) - } - return cfg.Passwd -} - -func TestWithPasswordBasic(t *testing.T) { - out, err := WithPassword("bts@tcp(127.0.0.1:55441)/", "s3cr3t") - if err != nil { - t.Fatal(err) - } - if pw := mustPassword(t, out); pw != "s3cr3t" { - t.Fatalf("password = %q, want s3cr3t", pw) - } - cfg, _ := mysql.ParseDSN(out) - if cfg.User != "bts" { - t.Fatalf("user = %q, want bts (must be preserved)", cfg.User) - } -} - -// go-sql-driver writes the password verbatim; a token full of DSN-significant -// characters (as an RDS-IAM token carries) must round-trip byte-for-byte. -func TestWithPasswordSpecialChars(t *testing.T) { - for _, tok := range []string{"p@ss:w/rd?x=1&y=2", "X-Amz=abc&Sig=d/e+f%20", `a'b\c d`} { - out, err := WithPassword("bts@tcp(h:3306)/db", tok) - if err != nil { - t.Fatalf("token %q: %v", tok, err) - } - if pw := mustPassword(t, out); pw != tok { - t.Fatalf("password = %q, want %q", pw, tok) - } - } -} - -func TestWithPasswordReplacesExisting(t *testing.T) { - out, err := WithPassword("bts:old@tcp(h:3306)/db", "new") - if err != nil { - t.Fatal(err) - } - if pw := mustPassword(t, out); pw != "new" { - t.Fatalf("password = %q, want new", pw) - } -} - -// The MySQL sharp edge: a userless DSN cannot carry a password (FormatDSN drops it), -// so WithPassword must refuse rather than silently connect without one. -func TestWithPasswordUserlessDSNFails(t *testing.T) { - _, err := WithPassword("tcp(127.0.0.1:55441)/", "orphan") - if err == nil { - t.Fatal("expected an error for a userless DSN (password would be silently dropped)") - } -} diff --git a/internal/storage/mysqldialect/driver.go b/internal/storage/mysqldialect/driver.go deleted file mode 100644 index 2b9c286db..000000000 --- a/internal/storage/mysqldialect/driver.go +++ /dev/null @@ -1,91 +0,0 @@ -// Package mysqldialect is the MySQL backend's thin driver wrapper. bd's canonical SQL -// is MySQL, so unlike pgdialect there is no translation — the ONE rewrite is the -// self-referential is_blocked recompute UPDATE, which MySQL 8 rejects (error 1093: a -// table may not be updated while referenced in its own subquery) but Dolt and Postgres -// permit. The rewrite is isolated here, so the shared issueops/sqlkit layers and the -// Dolt/Postgres backends are entirely untouched. -// -// A portable, perf-reviewed rewrite in shared issueops is tracked as a backlog item; -// this wrapper is the per-backend containment chosen to avoid touching Dolt's hot path. -package mysqldialect - -import ( - "context" - "database/sql" - "database/sql/driver" - - "github.com/go-sql-driver/mysql" -) - -// Open returns a *sql.DB over go-sql-driver whose Prepare rewrites the recompute -// UPDATE (see rewriteSelfRefUpdate); every other statement passes through verbatim. -// Like pgdialect, the wrapper exposes no Execer/Queryer fast path, so database/sql -// routes every statement through Prepare — the one place the rewrite happens. -func Open(dsn string) (*sql.DB, error) { - base, err := (mysql.MySQLDriver{}).OpenConnector(dsn) - if err != nil { - return nil, err - } - return sql.OpenDB(&connector{base: base}), nil -} - -type connector struct{ base driver.Connector } - -func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { - cn, err := c.base.Connect(ctx) - if err != nil { - return nil, err - } - return &conn{base: cn}, nil -} - -func (c *connector) Driver() driver.Driver { return c.base.Driver() } - -// conn wraps the mysql driver.Conn and rewrites the recompute UPDATE at prepare time. -// It deliberately implements no Execer/Queryer so database/sql prepares every stmt. -type conn struct{ base driver.Conn } - -func (c *conn) Prepare(query string) (driver.Stmt, error) { - return c.base.Prepare(rewriteSelfRefUpdate(query)) -} - -func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { - q := rewriteSelfRefUpdate(query) - if p, ok := c.base.(driver.ConnPrepareContext); ok { - return p.PrepareContext(ctx, q) - } - return c.base.Prepare(q) -} - -func (c *conn) Close() error { return c.base.Close() } - -//nolint:staticcheck // SA1019: driver.Conn requires the deprecated Begin; BeginTx forwarded below. -func (c *conn) Begin() (driver.Tx, error) { return c.base.Begin() } - -func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - if b, ok := c.base.(driver.ConnBeginTx); ok { - return b.BeginTx(ctx, opts) - } - return c.base.Begin() //nolint:staticcheck -} - -func (c *conn) ResetSession(ctx context.Context) error { - if rs, ok := c.base.(driver.SessionResetter); ok { - return rs.ResetSession(ctx) - } - return nil -} - -func (c *conn) IsValid() bool { - if v, ok := c.base.(driver.Validator); ok { - return v.IsValid() - } - return true -} - -func (c *conn) Ping(ctx context.Context) error { - if p, ok := c.base.(driver.Pinger); ok { - return p.Ping(ctx) - } - return nil -} diff --git a/internal/storage/mysqldialect/rewrite.go b/internal/storage/mysqldialect/rewrite.go deleted file mode 100644 index 500bced7c..000000000 --- a/internal/storage/mysqldialect/rewrite.go +++ /dev/null @@ -1,43 +0,0 @@ -package mysqldialect - -import ( - "fmt" - "regexp" - "strings" -) - -// selfRefUpdateRe matches bd's is_blocked recompute shape: -// -// UPDATE
SET <...> WHERE .id IN () -// -// The SET is non-greedy up to the first WHERE (SET clauses contain no WHERE); the -// placeholder list is the batched `?,?,…`. -var selfRefUpdateRe = regexp.MustCompile(`(?is)^(\s*UPDATE\s+(\w+)\s+(\w+)\s+SET\s+.+?)\s+WHERE\s+(\w+)\.id\s+IN\s+\(([^)]*)\)\s*(.*)$`) - -// rewriteSelfRefUpdate rewrites the is_blocked recompute UPDATE into a MySQL-8-safe -// double-nested form so the target table's self-reference is materialized (avoiding -// error 1093). Everything else is returned verbatim. The transform is semantics- -// preserving: the id set (batch ∩ conditions) is computed in a materialized derived -// table, and the outer UPDATE writes exactly those rows. Positional placeholders keep -// their order because the WHERE body is moved intact. -func rewriteSelfRefUpdate(q string) string { - // Fast reject: only the is_blocked recompute both sets is_blocked and self-joins. - if !strings.Contains(q, "is_blocked") { - return q - } - m := selfRefUpdateRe.FindStringSubmatch(q) - if m == nil { - return q - } - head, table, alias, whereAlias, placeholders, rest := m[1], m[2], m[3], m[4], m[5], m[6] - // Only the recompute has further conditions after the IN clause (the self-ref - // EXISTS block); a plain `UPDATE … WHERE id IN (…)` needs no rewrite. - if whereAlias != alias || strings.TrimSpace(rest) == "" { - return q - } - // Rebind the outer alias to the derived-table alias inside the moved conditions. - reb := regexp.MustCompile(`\b` + regexp.QuoteMeta(alias) + `\.`) - restRebound := reb.ReplaceAllString(rest, "_b.") - return fmt.Sprintf("%s WHERE %s.id IN (SELECT id FROM (SELECT _b.id FROM %s _b WHERE _b.id IN (%s) %s) AS _blk)", - head, alias, table, placeholders, restRebound) -} diff --git a/internal/storage/mysqldialect/rewrite_test.go b/internal/storage/mysqldialect/rewrite_test.go deleted file mode 100644 index 3e6357f29..000000000 --- a/internal/storage/mysqldialect/rewrite_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package mysqldialect - -import ( - "regexp" - "strings" - "testing" - - "github.com/steveyegge/beads/internal/storage/issueops" -) - -func TestRewriteSelfRefUpdate(t *testing.T) { - recompute := `UPDATE issues i SET i.is_blocked = 1, i.updated_at = i.updated_at - WHERE i.id IN (?,?,?) - AND i.is_blocked = 0 - AND i.status <> 'closed' AND i.status <> 'pinned' - AND (EXISTS (SELECT 1 FROM dependencies d JOIN issues t ON t.id = d.depends_on_issue_id - WHERE d.issue_id = i.id AND d.type = 'blocks' AND t.status <> 'closed'))` - - got := rewriteSelfRefUpdate(recompute) - if got == recompute { - t.Fatal("recompute UPDATE was not rewritten") - } - // The outer target must no longer be referenced directly in a subquery: the - // materialized derived table carries the self-reference. - if !strings.Contains(got, "SELECT id FROM (SELECT _b.id FROM issues _b") { - t.Fatalf("missing double-nested derived table:\n%s", got) - } - // The correlation must be rebound to the derived-table alias. - if !strings.Contains(got, "d.issue_id = _b.id") { - t.Fatalf("alias not rebound to _b:\n%s", got) - } - // The outer SET still targets the real alias. - if !strings.HasPrefix(strings.TrimSpace(got), "UPDATE issues i SET i.is_blocked = 1") { - t.Fatalf("outer UPDATE target changed:\n%s", got) - } - // Placeholder count preserved (positional binding). - if a, b := strings.Count(recompute, "?"), strings.Count(got, "?"); a != b { - t.Fatalf("placeholder count changed: %d -> %d", a, b) - } -} - -// TestRewriteMatchesRealRecomputeTemplates drives the rewrite from the ACTUAL -// issueops is_blocked recompute templates, not a hand-copied fixture. The rewrite -// regex is coupled to the exact shape those templates emit (UPDATE SET -// … WHERE .id IN (…) ); if a template later grows a predicate -// before `id IN`, backticks the table, renames the alias, or changes whitespace, the -// regex would silently stop matching and MySQL 8 would raise error 1093 at runtime on -// a core write path. That failure would otherwise only surface under the env-gated -// MySQL conformance suite — this non-gated test makes template drift fail `go test`. -func TestRewriteMatchesRealRecomputeTemplates(t *testing.T) { - templates := issueops.SelfReferentialRecomputeTemplates() - if len(templates) == 0 { - t.Fatal("issueops.SelfReferentialRecomputeTemplates() returned no templates") - } - // Parse the UPDATE target table+alias so the assertions can target the OUTER - // self-reference specifically (the inner materialized `_b.id IN (?…)` is expected). - updateHead := regexp.MustCompile(`(?is)^\s*UPDATE\s+(\w+)\s+(\w+)\s+SET\b`) - for i, tmpl := range templates { - m := updateHead.FindStringSubmatch(tmpl) - if m == nil { - t.Errorf("template[%d] is not an `UPDATE
SET …` shape:\n%s", i, tmpl) - continue - } - table, alias := m[1], m[2] - - got := rewriteSelfRefUpdate(tmpl) - if got == tmpl { - t.Errorf("template[%d] was NOT rewritten — the rewrite regex no longer matches an is_blocked "+ - "recompute template; MySQL error 1093 would return at runtime:\n%s", i, tmpl) - continue - } - // The target table must be materialized in a derived table so the outer UPDATE - // no longer references it directly in a subquery (the 1093 avoidance). - if want := "SELECT id FROM (SELECT _b.id FROM " + table + " _b"; !strings.Contains(got, want) { - t.Errorf("template[%d] rewrite missing the double-nested derived table %q:\n%s", i, want, got) - } - // The OUTER target alias must now select from the materialized set, not a bare - // self-referential `.id IN (?…)`. - outerMaterialized := regexp.MustCompile(`(?i)\b` + alias + `\.id\s+IN\s+\(\s*SELECT`) - if !outerMaterialized.MatchString(got) { - t.Errorf("template[%d] outer %s.id IN (…) was not materialized into a subselect:\n%s", i, alias, got) - } - outerRaw := regexp.MustCompile(`(?i)\b` + alias + `\.id\s+IN\s+\(\s*\?`) - if outerRaw.MatchString(got) { - t.Errorf("template[%d] outer %s.id IN (?…) is still a raw self-reference after rewrite (MySQL 1093):\n%s", i, alias, got) - } - // Positional binding is preserved (placeholders unchanged). - if a, b := strings.Count(tmpl, "?"), strings.Count(got, "?"); a != b { - t.Errorf("template[%d] placeholder count changed: %d -> %d", i, a, b) - } - } -} - -func TestRewritePassThrough(t *testing.T) { - cases := []string{ - "SELECT * FROM issues WHERE id = ?", - "UPDATE issues SET title = ? WHERE id = ?", // no is_blocked, no IN - "UPDATE issues i SET i.is_blocked = 1 WHERE i.id IN (?,?)", // batch update, no self-ref conditions - "INSERT INTO issues (id) VALUES (?)", - "DELETE FROM issues WHERE id = ?", - } - for _, q := range cases { - if got := rewriteSelfRefUpdate(q); got != q { - t.Errorf("query was rewritten but should pass through:\n in: %s\nout: %s", q, got) - } - } -} - -// The rewrite must not leave the outer target table referenced in a subquery in a way -// that regex could double-apply; a second pass is a no-op on already-rewritten SQL. -func TestRewriteIdempotentShape(t *testing.T) { - q := `UPDATE wisps w SET w.is_blocked = 0, w.updated_at = w.updated_at - WHERE w.id IN (?) AND w.is_blocked = 1 AND (w.status = 'closed')` - once := rewriteSelfRefUpdate(q) - if once == q { - t.Fatal("expected rewrite") - } - // A plain "?,?" derived form should still parse-shape sanely (no target alias left - // as a bare self-join outside the derived table). - if regexp.MustCompile(`WHERE w\.id IN \(\?`).MatchString(once) { - t.Fatalf("outer WHERE still has a raw placeholder IN (self-ref not materialized):\n%s", once) - } -} diff --git a/internal/storage/pgdialect/corpus_test.go b/internal/storage/pgdialect/corpus_test.go deleted file mode 100644 index 4e95c7d35..000000000 --- a/internal/storage/pgdialect/corpus_test.go +++ /dev/null @@ -1,430 +0,0 @@ -package pgdialect_test - -// This file is the PREPARE gate the package doc promises. It enumerates every -// distinct statement family bd's shared layer (sqlbuild + issueops) emits on -// the read/search path, runs each through pgdialect.Translate, and — when a -// live Postgres is configured via BEADS_PG_TEST_URL — PREPAREs the translated -// text against a scratch schema. An untranslatable construct fails the test -// loudly (Translate error, surviving MySQL residue, or a PREPARE-time Postgres -// error) instead of silently mistranslating a row set. -// -// It is an external test package (pgdialect_test) so it can live alongside the -// white-box translate_test.go while only touching pgdialect's exported surface. -// -// The scratch schema is built here rather than via postgres.InitSchema on -// purpose: pgdialect is a lower layer than the postgres backend (postgres -// imports pgdialect), so importing postgres into pgdialect's tests would invert -// the dependency and couple this gate to that package's build state. The tables -// are generated from sqlbuild.IssueSelectColumns so they cannot drift out of -// sync with the column list the queries project. Only the shapes the corpus -// exercises are created; types are faithful where an operator depends on them -// (jsonb metadata, integer priority, smallint flags, timestamp columns) and -// text elsewhere. - -import ( - "context" - "database/sql" - "fmt" - "os" - "strings" - "testing" - "time" - - "github.com/steveyegge/beads/internal/storage/pgdialect" - "github.com/steveyegge/beads/internal/storage/sqlbuild" - "github.com/steveyegge/beads/internal/types" -) - -type corpusStmt struct { - name string - sql string -} - -func strp(s string) *string { return &s } -func intp(i int) *int { return &i } - -// searchSelectSQL mirrors issueops.searchTableInTx Pattern A: the full-column -// filtered scan for one table family (incl. the label-driven JOIN plan). -func searchSelectSQL(tables sqlbuild.FilterTables, query string, filter types.IssueFilter) (string, error) { - plan := sqlbuild.BuildLabelDrivenSearch(filter, tables) - where, _, err := sqlbuild.BuildIssueFilterClauses(query, plan.Filter, tables) - if err != nil { - return "", err - } - where, _ = plan.MergeInto(where, nil) - whereSQL := "" - if len(where) > 0 { - whereSQL = "WHERE " + strings.Join(where, " AND ") - } - selectKw := "SELECT " - if plan.Distinct { - selectKw = "SELECT DISTINCT " - } - return fmt.Sprintf("%s%s FROM %s %s %s %s", - selectKw, sqlbuild.IssueSelectColumns, plan.FromSQL, sqlbuild.LeaseJoin(tables.Main), whereSQL, - sqlbuild.OrderBy(filter.SortBy, filter.SortDesc, "")), nil -} - -// searchCountsSQL mirrors issueops.runFilterSearchQueryInTx: the counts -// mega-query (the DATE_FORMAT / JSON_OBJECT / CAST / JSON_ARRAYAGG surface). -func searchCountsSQL(tables sqlbuild.FilterTables, query string, filter types.IssueFilter, includeWispReverseDeps bool) (string, error) { - where, _, err := sqlbuild.BuildIssueFilterClauses(query, filter, tables) - if err != nil { - return "", err - } - whereSQL := "" - if len(where) > 0 { - whereSQL = "WHERE " + strings.Join(where, " AND ") - } - sql, _ := sqlbuild.SearchCountsSQL(tables, nil, whereSQL, sqlbuild.OrderBy(filter.SortBy, filter.SortDesc, "i"), "", includeWispReverseDeps, filter.SkipLabels) - return sql, nil -} - -// readySelectSQL mirrors issueops.GetReadyWorkInTx: SELECT id FROM
with -// the ready-work WHERE/ORDER. -func readySelectSQL(tables sqlbuild.FilterTables, filter types.WorkFilter) (string, error) { - whereSQL, _, err := sqlbuild.BuildReadyWorkWhere(filter, tables, sqlbuild.ReadyWorkWhereInputs{}) - if err != nil { - return "", err - } - order := sqlbuild.BuildReadyWorkOrder(filter.SortPolicy, "created_at", "priority") - return fmt.Sprintf("SELECT id FROM %s %s %s", tables.Main, whereSQL, order.SQL), nil -} - -// readyCountsSQL mirrors issueops.GetReadyWorkWithCountsInTx: the counts -// mega-query fed the ready-work predicates. -func readyCountsSQL(tables sqlbuild.FilterTables, filter types.WorkFilter, includeWispReverseDeps bool) (string, error) { - whereSQL, _, err := sqlbuild.BuildReadyWorkWhere(filter, tables, sqlbuild.ReadyWorkWhereInputs{}) - if err != nil { - return "", err - } - order := sqlbuild.BuildReadyWorkOrder(filter.SortPolicy, "created_at", "priority") - sql, _ := sqlbuild.SearchCountsSQL(tables, nil, whereSQL, order.SQL, "", includeWispReverseDeps, false) - return sql, nil -} - -// adaptiveIDInstrSQL is issueops.GetAdaptiveIDLengthTx's create-path query -// (CONCAT + INSTR + SUBSTRING). Reproduced verbatim so the corpus covers the -// non-search statement family the smoke test's explicit-ID creates never hit. -const adaptiveIDInstrSQL = `SELECT COUNT(*) - FROM issues - WHERE id LIKE CONCAT(?, '-%') - AND INSTR(SUBSTRING(id, LENGTH(?) + 2), '.') = 0` - -func buildCorpus(t *testing.T) []corpusStmt { - t.Helper() - - // Representative filters: priority, an AND label, metadata has-key + eq, - // and a parent filter (each drives a distinct clause shape). - richIssue := types.IssueFilter{ - Priority: intp(1), - Labels: []string{"backlog"}, - HasMetadataKey: "sprint", - MetadataFields: map[string]string{"team": "core"}, - ParentID: strp("bd-1"), - SortBy: "priority", - } - richWork := types.WorkFilter{ - Status: types.Status("open"), - Priority: intp(1), - Labels: []string{"backlog"}, - HasMetadataKey: "sprint", - MetadataFields: map[string]string{"team": "core"}, - ParentID: strp("bd-1"), - } - metaOnly := types.IssueFilter{HasMetadataKey: "sprint", MetadataFields: map[string]string{"team": "core"}} - - var corpus []corpusStmt - add := func(name, sqlText string, err error) { - if err != nil { - t.Fatalf("build corpus %q: %v", name, err) - } - corpus = append(corpus, corpusStmt{name: name, sql: sqlText}) - } - - s, err := searchSelectSQL(sqlbuild.IssuesFilterTables, "", types.IssueFilter{}) - add("search/issues/plain", s, err) - s, err = searchSelectSQL(sqlbuild.IssuesFilterTables, "bd-1", richIssue) - add("search/issues/rich", s, err) - s, err = searchSelectSQL(sqlbuild.WispsFilterTables, "", richIssue) - add("search/wisps/rich", s, err) - s, err = searchSelectSQL(sqlbuild.IssuesFilterTables, "", metaOnly) - add("search/issues/metadata-only", s, err) - - s, err = searchCountsSQL(sqlbuild.IssuesFilterTables, "", types.IssueFilter{}, false) - add("counts/issues/plain", s, err) - s, err = searchCountsSQL(sqlbuild.IssuesFilterTables, "", richIssue, true) - add("counts/issues/rich", s, err) - s, err = searchCountsSQL(sqlbuild.WispsFilterTables, "", richIssue, true) - add("counts/wisps/rich", s, err) - - s, err = readySelectSQL(sqlbuild.IssuesFilterTables, richWork) - add("ready/issues", s, err) - s, err = readySelectSQL(sqlbuild.WispsFilterTables, richWork) - add("ready/wisps", s, err) - s, err = readyCountsSQL(sqlbuild.IssuesFilterTables, richWork, true) - add("ready-counts/issues", s, err) - s, err = readyCountsSQL(sqlbuild.WispsFilterTables, richWork, true) - add("ready-counts/wisps", s, err) - - corpus = append(corpus, - corpusStmt{"adaptive-id/instr", adaptiveIDInstrSQL}, - // Literal-path gate check (issueops/blocked_state.go): keeps the literal - // JSON rule covered alongside the dynamic one. - corpusStmt{"metadata/literal-gate", `SELECT id FROM issues WHERE JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.gate')) = ?`}, - // Dep JSON aggregation in isolation (DATE_FORMAT + JSON_OBJECT + CAST). - corpusStmt{"dep-json/arrayagg", fmt.Sprintf(`SELECT issue_id, JSON_ARRAYAGG(%s) AS deps_json FROM dependencies GROUP BY issue_id`, sqlbuild.DepJSONObject)}, - ) - return corpus -} - -// forbiddenResidue are lowercased markers of MySQL constructs that must never -// survive translation. JSON_ARRAYAGG is intentionally absent: Postgres 16 -// supports SQL-standard JSON_ARRAYAGG natively, so the wedge leaves it in place. -var forbiddenResidue = []string{ - "date_format(", "json_object(", "json_unquote(", "json_extract(", - " as char", "cast(", "insert ignore", "replace into", "on duplicate", - "utc_timestamp", "now()", "`", "?", -} - -func assertNoResidue(t *testing.T, name, translated string) { - t.Helper() - low := strings.ToLower(translated) - for _, marker := range forbiddenResidue { - if strings.Contains(low, marker) { - t.Errorf("%s: untranslated MySQL residue %q survived:\n%s", name, strings.TrimSpace(marker), translated) - } - } -} - -// TestCorpusTranslates runs with no database: it proves every statement family -// translates without error and leaves no MySQL residue. The fail-loud property -// is a build-time guarantee, so this half of the gate always runs. -func TestCorpusTranslates(t *testing.T) { - for _, c := range buildCorpus(t) { - translated, err := pgdialect.Translate(c.sql) - if err != nil { - t.Errorf("%s: Translate error: %v\n-- sql:\n%s", c.name, err, c.sql) - continue - } - assertNoResidue(t, c.name, translated) - } -} - -// TestCorpusPrepare is the live half: PREPARE every translated statement against -// a real Postgres so a construct that is syntactically valid but references a -// missing column/function/operator still fails loudly. Gated on BEADS_PG_TEST_URL. -func TestCorpusPrepare(t *testing.T) { - dsn := os.Getenv("BEADS_PG_TEST_URL") - if dsn == "" { - t.Skip("BEADS_PG_TEST_URL not set; skipping live Postgres PREPARE gate") - } - ctx := context.Background() - schema := fmt.Sprintf("corpus_%d", time.Now().UnixNano()) - - db, err := pgdialect.Open(dsn, schema) - if err != nil { - t.Fatalf("pgdialect.Open: %v", err) - } - t.Cleanup(func() { - _, _ = db.ExecContext(context.Background(), fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) - _ = db.Close() - }) - if err := createScratchSchema(ctx, db, schema); err != nil { - t.Fatalf("create scratch schema: %v", err) - } - - for _, c := range buildCorpus(t) { - t.Run(c.name, func(t *testing.T) { - translated, err := pgdialect.Translate(c.sql) - if err != nil { - t.Fatalf("Translate error: %v\n-- sql:\n%s", err, c.sql) - } - assertNoResidue(t, c.name, translated) - // PREPARE the RAW statement through the translating driver — the exact - // production path (driver.Translate then server Parse). - stmt, err := db.PrepareContext(ctx, c.sql) - if err != nil { - t.Fatalf("PREPARE failed: %v\n-- translated:\n%s", err, translated) - } - _ = stmt.Close() - }) - } -} - -// createScratchSchema builds just enough of the wedge schema for the corpus to -// PREPARE against: the issues/wisps families (generated from IssueSelectColumns -// so they track the projected column list), their label/dependency/comment -// side tables, and the bd_mysql_jsonkey helper the dynamic-path rewrite calls -// (in production schema.go/InitSchema owns this function). -func createScratchSchema(ctx context.Context, db *sql.DB, schema string) error { - stmts := []string{ - fmt.Sprintf(`CREATE SCHEMA IF NOT EXISTS %q`, schema), - issueTableDDL("issues"), - issueTableDDL("wisps"), - `CREATE TABLE leases (issue_id text, holder text, granted_at timestamp(0), - lease_expires_at timestamp(0), heartbeat_at timestamp(0))`, - `CREATE TABLE labels (issue_id text, label text)`, - `CREATE TABLE wisp_labels (issue_id text, label text)`, - depTableDDL("dependencies"), - depTableDDL("wisp_dependencies"), - `CREATE TABLE comments (issue_id text)`, - `CREATE TABLE wisp_comments (issue_id text)`, - `CREATE OR REPLACE FUNCTION bd_mysql_jsonkey(p text) RETURNS text LANGUAGE sql IMMUTABLE AS $$ - SELECT CASE WHEN p LIKE '$."%"' THEN substring(p from 4 for length(p)-4) ELSE substring(p from 3) END - $$`, - } - for _, s := range stmts { - if _, err := db.ExecContext(ctx, s); err != nil { - return fmt.Errorf("exec %q: %w", firstLine(s), err) - } - } - return nil -} - -// issueTableDDL renders a CREATE TABLE for the issues/wisps family from -// IssueBaseColumns plus the derived is_blocked column the ready-work WHERE -// reads. The lease overlay columns live in the separate leases table -// (bd-lrgn1), created explicitly in createScratchSchema. Column types are -// faithful where an operator depends on them. -func issueTableDDL(table string) string { - var defs []string - for _, name := range splitColumnList(sqlbuild.IssueBaseColumns) { - defs = append(defs, name+" "+pgColumnType(name)) - } - defs = append(defs, "is_blocked smallint NOT NULL DEFAULT 0") - return fmt.Sprintf("CREATE TABLE %s (%s)", table, strings.Join(defs, ", ")) -} - -func depTableDDL(table string) string { - return fmt.Sprintf(`CREATE TABLE %s ( - issue_id text, depends_on_issue_id text, depends_on_wisp_id text, - depends_on_external text, type text, created_by text, - created_at timestamp(0), thread_id text, metadata jsonb)`, table) -} - -func pgColumnType(name string) string { - switch name { - case "metadata": - return "jsonb" - case "created_at", "updated_at", "started_at", "closed_at", "compacted_at", "due_at", "defer_until": - return "timestamp(0)" - case "priority", "estimated_minutes", "compaction_level", "original_size", "timeout_ns": - return "integer" - case "pinned", "ephemeral", "is_template", "no_history": - return "smallint" - default: - return "text" - } -} - -// splitColumnList splits a select-column constant into bare column names. -func splitColumnList(cols string) []string { - raw := strings.NewReplacer("\n", " ", "\t", " ").Replace(cols) - var names []string - for _, p := range strings.Split(raw, ",") { - if n := strings.TrimSpace(p); n != "" { - names = append(names, n) - } - } - return names -} - -func firstLine(s string) string { - if i := strings.IndexByte(s, '\n'); i >= 0 { - return s[:i] - } - return s -} - -// TestTranslateNewRules pins the four red-team translator rules (DATE_FORMAT, -// JSON_OBJECT comma form, CAST AS CHAR, dynamic ?-path JSON metadata), including -// their fail-loud (left-untranslated) branches. -func TestTranslateNewRules(t *testing.T) { - cases := []struct { - name string - in string - want string - }{ - { - name: "DATE_FORMAT ISO8601 -> to_char", - in: `SELECT DATE_FORMAT(created_at, '%Y-%m-%dT%H:%i:%sZ') FROM issues`, - want: `SELECT to_char(created_at, 'YYYY-MM-DD"T"HH24:MI:SS"Z"') FROM issues`, - }, - { - name: "DATE_FORMAT unknown format left loud", - in: `SELECT DATE_FORMAT(created_at, '%Y-%m-%d') FROM issues`, - want: `SELECT DATE_FORMAT(created_at, '%Y-%m-%d') FROM issues`, - }, - { - name: "JSON_OBJECT even -> jsonb_build_object", - in: `SELECT JSON_OBJECT('a', x, 'b', y) FROM t`, - want: `SELECT jsonb_build_object('a', x, 'b', y) FROM t`, - }, - { - name: "JSON_OBJECT odd arg count left loud", - in: `SELECT JSON_OBJECT('a', x, 'b') FROM t`, - want: `SELECT JSON_OBJECT('a', x, 'b') FROM t`, - }, - { - name: "CAST AS CHAR -> ::text", - in: `SELECT CAST(metadata AS CHAR) FROM t`, - want: `SELECT (metadata)::text FROM t`, - }, - { - name: "CAST AS CHAR(n) -> ::text", - in: `SELECT CAST(metadata AS CHAR(64)) FROM t`, - want: `SELECT (metadata)::text FROM t`, - }, - { - name: "CAST AS SIGNED left loud", - in: `SELECT CAST(x AS SIGNED) FROM t`, - want: `SELECT CAST(x AS SIGNED) FROM t`, - }, - { - name: "dynamic has-key JSON_EXTRACT -> -> bd_mysql_jsonkey", - in: `SELECT id FROM issues WHERE JSON_EXTRACT(metadata, ?) IS NOT NULL`, - want: `SELECT id FROM issues WHERE (metadata -> bd_mysql_jsonkey($1)) IS NOT NULL`, - }, - { - name: "dynamic eq JSON_UNQUOTE(JSON_EXTRACT) -> ->> bd_mysql_jsonkey", - in: `SELECT id FROM issues WHERE JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) = ?`, - want: `SELECT id FROM issues WHERE (metadata ->> bd_mysql_jsonkey($1)) = $2`, - }, - { - name: "literal path rule still applies (regression)", - in: `SELECT 1 WHERE JSON_UNQUOTE(JSON_EXTRACT(d.metadata, '$.gate')) = 'any-children'`, - want: `SELECT 1 WHERE (d.metadata #>> '{gate}') = 'any-children'`, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got, err := pgdialect.Translate(tc.in) - if err != nil { - t.Fatalf("Translate error: %v", err) - } - if got != tc.want { - t.Errorf("mismatch\n in: %s\n got: %s\n want: %s", tc.in, got, tc.want) - } - }) - } -} - -// TestTranslateDepJSONObject exercises the real counts.go DepJSONObject constant -// (DATE_FORMAT + JSON_OBJECT + CAST composed) end to end. -func TestTranslateDepJSONObject(t *testing.T) { - got, err := pgdialect.Translate(sqlbuild.DepJSONObject) - if err != nil { - t.Fatalf("Translate error: %v", err) - } - for _, want := range []string{ - "jsonb_build_object(", - `to_char(created_at, 'YYYY-MM-DD"T"HH24:MI:SS"Z"')`, - "(metadata)::text", - } { - if !strings.Contains(got, want) { - t.Errorf("DepJSONObject translation missing %q:\n%s", want, got) - } - } - assertNoResidue(t, "DepJSONObject", got) -} diff --git a/internal/storage/pgdialect/credential.go b/internal/storage/pgdialect/credential.go deleted file mode 100644 index d3b04c062..000000000 --- a/internal/storage/pgdialect/credential.go +++ /dev/null @@ -1,121 +0,0 @@ -package pgdialect - -import ( - "fmt" - "net/url" - "regexp" - "strings" - - "github.com/jackc/pgx/v5" -) - -// kvUserRe matches a libpq keyword/value user token, whose value is either -// single-quoted or a run of non-space chars. Like kvPasswordRe (redact.go) it is a -// token-level regex, not a full parser, so a ` user=` sequence inside another token's -// quoted value could be matched — an exotic case (a keyword-form DSN whose quoted -// value contains ` user=`, combined with a username-bearing credential). The -// round-trip verify in WithCredential catches the loud failures; the silent case is a -// documented sharp edge shared with kvPasswordRe. -var kvUserRe = regexp.MustCompile(`(?i)(^|\s)user\s*=\s*(?:'(?:[^'\\]|\\.)*'|\S*)`) - -// kvHasPasswordRe detects a libpq keyword/value connection password token (password=, -// not sslpassword=) for the DSN-embedded check in HasPassword. -var kvHasPasswordRe = regexp.MustCompile(`(?i)(^|\s)password\s*=`) - -// HasPassword reports whether the connection string itself EMBEDS a password — URL -// userinfo, a URL ?password= query param, or a libpq password= token. It deliberately -// does NOT report whether pgx would resolve a password from the ambient environment: -// pgx.ParseConfig folds PGPASSWORD and ~/.pgpass into cfg.Password, so parsing here -// would let a stale PGPASSWORD short-circuit the whole ladder and mask a configured -// command or credentials file. An embedded password is the only thing that wins -// outright; ambient PGPASSWORD/~/.pgpass remain the driver-native fallback, consulted -// at connect only when no ladder rung is configured. -func HasPassword(dsn string) bool { - if u, err := url.Parse(dsn); err == nil && u.Scheme != "" { - if u.User != nil { - if _, ok := u.User.Password(); ok { - return true - } - } - return u.Query().Get("password") != "" - } - return kvHasPasswordRe.MatchString(dsn) -} - -// WithCredential returns dsn with the resolved credential placed: the password -// always, and the username only when non-empty (a Vault-style dynamic user/password -// pair) — an empty username leaves the DSN's user untouched. It handles both grammars -// pgx accepts (URL userinfo for scheme:// DSNs, single-quoted keyword/value tokens -// otherwise) and VERIFIES with the same parser pgx uses that both fields round-tripped, -// failing loudly rather than emit a DSN that silently drops or mangles the secret. -func WithCredential(dsn, username, password string) (string, error) { - var out string - if u, err := url.Parse(dsn); err == nil && u.Scheme != "" { - // URL form: set the userinfo. url.String() percent-encodes special characters - // (an RDS-IAM token's '&'/'='/'/'), which pgx.ParseConfig decodes back — the - // verify below proves it. - user := username - if user == "" && u.User != nil { - user = u.User.Username() - } - u.User = url.UserPassword(user, password) - out = u.String() - } else { - // libpq keyword/value form: drop any existing password token (and, when we are - // overriding the user, the user token too), then append single-quoted tokens. - stripped := kvPasswordRe.ReplaceAllString(dsn, "$1") - if username != "" { - stripped = kvUserRe.ReplaceAllString(stripped, "$1") - } - stripped = strings.TrimSpace(strings.Join(strings.Fields(stripped), " ")) - if username != "" { - stripped += " user='" + escapeKeywordValue(username) + "'" - } - out = stripped + " password='" + escapeKeywordValue(password) + "'" - } - - cfg, err := pgx.ParseConfig(out) - if err != nil { - return "", fmt.Errorf("cannot place credential into the connection string (unparseable result): %w", err) - } - if cfg.Password != password { - return "", fmt.Errorf("password did not round-trip into the connection string; refusing to connect with a wrong or empty password") - } - if username != "" && cfg.User != username { - return "", fmt.Errorf("username did not round-trip into the connection string; refusing to connect as the wrong user") - } - return out, nil -} - -// WithPassword places only the password, preserving the DSN's user. It is -// WithCredential with no username override. This is the counterpart to -// RedactPassword: RedactPassword strips the password for persistence, WithPassword -// re-injects the resolved password at open time. -func WithPassword(dsn, password string) (string, error) { - return WithCredential(dsn, "", password) -} - -// HostPort returns the TCP endpoint pgx would connect to for dsn, for keying the -// [host:port] credentials-file lookup. ok=false when the DSN does not parse or -// resolves to a unix-socket directory (an absolute-path host) — a peer-auth socket -// has no place in a password file, and the caller degrades the file rung to -// not-configured. For a multi-host DSN this is the primary endpoint only. IPv6 hosts -// are returned bare (e.g. "::1"); the credentials-file key format does not bracket -// them, so an IPv6 endpoint effectively degrades to not-found in the file. -func HostPort(dsn string) (host string, port int, ok bool) { - cfg, err := pgx.ParseConfig(dsn) - if err != nil { - return "", 0, false - } - if cfg.Host == "" || strings.HasPrefix(cfg.Host, "/") { - return "", 0, false - } - return cfg.Host, int(cfg.Port), true -} - -// escapeKeywordValue escapes a value for a single-quoted libpq keyword/value token. -func escapeKeywordValue(s string) string { - s = strings.ReplaceAll(s, `\`, `\\`) - s = strings.ReplaceAll(s, `'`, `\'`) - return s -} diff --git a/internal/storage/pgdialect/credential_extra_test.go b/internal/storage/pgdialect/credential_extra_test.go deleted file mode 100644 index 20251cd12..000000000 --- a/internal/storage/pgdialect/credential_extra_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package pgdialect - -import ( - "os" - "path/filepath" - "testing" - - "github.com/jackc/pgx/v5" -) - -// TestMain neutralizes ambient PGPASSWORD/~/.pgpass so redaction and HasPassword tests -// are hermetic (pgx.ParseConfig folds them into a connection). The ambient-immunity -// tests re-set PGPASSWORD locally via t.Setenv. -func TestMain(m *testing.M) { - dir, err := os.MkdirTemp("", "beads-pgdialect-test") - if err != nil { - panic(err) - } - os.Unsetenv("PGPASSWORD") - os.Setenv("PGPASSFILE", filepath.Join(dir, "no-pgpass")) - code := m.Run() - _ = os.RemoveAll(dir) - os.Exit(code) -} - -func mustUser(t *testing.T, out string) string { - t.Helper() - cfg, err := pgx.ParseConfig(out) - if err != nil { - t.Fatalf("pgx.ParseConfig(%q): %v", out, err) - } - return cfg.User -} - -func TestWithCredentialURLForm(t *testing.T) { - out, err := WithCredential("postgres://old@h:5432/db", "vu", "pw") - if err != nil { - t.Fatal(err) - } - if u := mustUser(t, out); u != "vu" { - t.Fatalf("user = %q, want vu", u) - } - if pw := mustPassword(t, out); pw != "pw" { - t.Fatalf("password = %q, want pw", pw) - } -} - -func TestWithCredentialURLEmptyUsernameKeepsUser(t *testing.T) { - out, err := WithCredential("postgres://old@h:5432/db", "", "pw") - if err != nil { - t.Fatal(err) - } - if u := mustUser(t, out); u != "old" { - t.Fatalf("user = %q, want old (preserved)", u) - } -} - -func TestWithCredentialKeywordForm(t *testing.T) { - out, err := WithCredential("host=h user=old dbname=db", "vu", "pw") - if err != nil { - t.Fatal(err) - } - if u := mustUser(t, out); u != "vu" { - t.Fatalf("user = %q, want vu (old stripped)", u) - } - if pw := mustPassword(t, out); pw != "pw" { - t.Fatalf("password = %q, want pw", pw) - } -} - -// WithCredential's own round-trip verify guards escaping; a user and password full of -// quotes/backslashes must survive the keyword grammar. -func TestWithCredentialKeywordEscaping(t *testing.T) { - wantU, wantP := `v'u\x`, `p'w\y` - out, err := WithCredential("host=h dbname=db", wantU, wantP) - if err != nil { - t.Fatal(err) - } - if u := mustUser(t, out); u != wantU { - t.Fatalf("user = %q, want %q", u, wantU) - } - if pw := mustPassword(t, out); pw != wantP { - t.Fatalf("password = %q, want %q", pw, wantP) - } -} - -func TestWithCredentialURLSpecialCharsPassword(t *testing.T) { - tok := "p@ss:w/rd&x=1" - out, err := WithCredential("postgres://h:5432/db", "dynuser", tok) - if err != nil { - t.Fatal(err) - } - if u := mustUser(t, out); u != "dynuser" { - t.Fatalf("user = %q, want dynuser", u) - } - if pw := mustPassword(t, out); pw != tok { - t.Fatalf("password = %q, want %q", pw, tok) - } -} - -func TestHostPort(t *testing.T) { - cases := []struct { - dsn string - host string - port int - ok bool - }{ - {"postgres://u@127.0.0.1:6543/db", "127.0.0.1", 6543, true}, - {"postgres://u@db.corp/db", "db.corp", 5432, true}, // pgx defaults the port to 5432 - {"host=db.corp user=u", "db.corp", 5432, true}, // keyword form - {"host=/var/run/postgresql user=u", "", 0, false}, // unix-socket dir -> not ok - } - for _, c := range cases { - h, p, ok := HostPort(c.dsn) - if ok != c.ok || (ok && (h != c.host || p != c.port)) { - t.Errorf("HostPort(%q) = (%q,%d,%v), want (%q,%d,%v)", c.dsn, h, p, ok, c.host, c.port, c.ok) - } - } -} - -// HasPassword must reflect only an embedded password, not one pgx would resolve from -// ambient PGPASSWORD/~/.pgpass — otherwise a stale PGPASSWORD would short-circuit the -// whole ladder. -func TestHasPasswordIgnoresAmbientPGPASSWORD(t *testing.T) { - t.Setenv("PGPASSWORD", "from-env") - if HasPassword("postgres://bts@127.0.0.1:5432/db") { - t.Fatal("HasPassword must not report a password from ambient PGPASSWORD") - } - if !HasPassword("postgres://bts:embedded@127.0.0.1:5432/db") { - t.Fatal("HasPassword must detect an embedded URL password") - } - if !HasPassword("postgres://bts@127.0.0.1:5432/db?password=q") { - t.Fatal("HasPassword must detect a ?password= query param") - } -} - -// RedactPassword must not be fooled by an ambient PGPASSWORD (which pgx would fold in -// but which is never persisted) into refusing a password-less --pg-url at init. -func TestRedactPasswordIgnoresAmbientPGPASSWORD(t *testing.T) { - t.Setenv("PGPASSWORD", "from-env") - got, err := RedactPassword("postgres://bts@127.0.0.1:5432/db") - if err != nil { - t.Fatalf("RedactPassword must not be fooled by ambient PGPASSWORD: %v", err) - } - if HasPassword(got) { - t.Fatalf("redacted DSN still carries an embedded password: %q", got) - } -} diff --git a/internal/storage/pgdialect/credential_test.go b/internal/storage/pgdialect/credential_test.go deleted file mode 100644 index 92f46f28c..000000000 --- a/internal/storage/pgdialect/credential_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package pgdialect - -import ( - "testing" - - "github.com/jackc/pgx/v5" -) - -func TestHasPassword(t *testing.T) { - cases := []struct { - dsn string - want bool - }{ - {"postgres://bts:secret@127.0.0.1:5432/db", true}, - {"postgres://bts@127.0.0.1:5432/db", false}, - {"host=127.0.0.1 port=5432 user=bts password=secret dbname=db", true}, - {"host=127.0.0.1 port=5432 user=bts dbname=db", false}, - } - for _, c := range cases { - if got := HasPassword(c.dsn); got != c.want { - t.Errorf("HasPassword(%q) = %v, want %v", c.dsn, got, c.want) - } - } -} - -// mustPassword parses out and returns the password pgx sees. -func mustPassword(t *testing.T, out string) string { - t.Helper() - cfg, err := pgx.ParseConfig(out) - if err != nil { - t.Fatalf("pgx.ParseConfig(%q): %v", out, err) - } - return cfg.Password -} - -func TestWithPasswordURLForm(t *testing.T) { - out, err := WithPassword("postgres://bts@127.0.0.1:5432/db", "s3cr3t") - if err != nil { - t.Fatal(err) - } - if pw := mustPassword(t, out); pw != "s3cr3t" { - t.Fatalf("password = %q, want s3cr3t", pw) - } - // The username must be preserved. - cfg, _ := pgx.ParseConfig(out) - if cfg.User != "bts" { - t.Fatalf("user = %q, want bts", cfg.User) - } -} - -// A token with URL-reserved characters (as an RDS-IAM / GCP-IAM auth token carries) -// must round-trip through the URL grammar intact. -func TestWithPasswordURLSpecialChars(t *testing.T) { - tok := "X-Amz=abc&Signature=d/e+f%20g" - out, err := WithPassword("postgres://bts@host:5432/db", tok) - if err != nil { - t.Fatal(err) - } - if pw := mustPassword(t, out); pw != tok { - t.Fatalf("password = %q, want %q", pw, tok) - } -} - -// The keyword/value grammar is the case the old net/url merge silently dropped. -func TestWithPasswordKeywordForm(t *testing.T) { - out, err := WithPassword("host=127.0.0.1 port=5432 user=bts dbname=db", "s3 cr3t") // note the space - if err != nil { - t.Fatal(err) - } - if pw := mustPassword(t, out); pw != "s3 cr3t" { - t.Fatalf("password = %q, want %q", pw, "s3 cr3t") - } -} - -// An existing password in a keyword DSN is replaced, not duplicated. -func TestWithPasswordKeywordReplacesExisting(t *testing.T) { - out, err := WithPassword("host=h user=u password=old dbname=db", "new") - if err != nil { - t.Fatal(err) - } - if pw := mustPassword(t, out); pw != "new" { - t.Fatalf("password = %q, want new", pw) - } -} - -// A value with a single quote and a backslash survives keyword escaping. -func TestWithPasswordKeywordEscaping(t *testing.T) { - tok := `a'b\c` - out, err := WithPassword("host=h user=u dbname=db", tok) - if err != nil { - t.Fatal(err) - } - if pw := mustPassword(t, out); pw != tok { - t.Fatalf("password = %q, want %q", pw, tok) - } -} diff --git a/internal/storage/pgdialect/driver.go b/internal/storage/pgdialect/driver.go deleted file mode 100644 index 65ea6b06a..000000000 --- a/internal/storage/pgdialect/driver.go +++ /dev/null @@ -1,190 +0,0 @@ -package pgdialect - -import ( - "context" - "database/sql" - "database/sql/driver" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/stdlib" -) - -// Open returns a *sql.DB over pgx whose Prepare/PrepareContext translate bd's -// canonical (MySQL-dialect) SQL to Postgres via Translate. searchPath pins the -// workspace schema. -// -// This wrapper deliberately exposes no Queryer/Execer fast path, so database/sql -// routes every statement through Prepare — translation therefore happens in -// exactly one place, and an untranslatable construct fails loudly at PREPARE. -func Open(dsn, searchPath string) (*sql.DB, error) { - cfg, err := pgx.ParseConfig(dsn) - if err != nil { - // pgx's ParseConfigError leaves ?password= URL query params cleartext; scrub - // before the error can reach any log. - return nil, ScrubDSNError(dsn, err) - } - if searchPath != "" { - if cfg.RuntimeParams == nil { - cfg.RuntimeParams = map[string]string{} - } - cfg.RuntimeParams["search_path"] = searchPath - } - return sql.OpenDB(&connector{base: stdlib.GetConnector(*cfg)}), nil -} - -// OpenRaw returns a *sql.DB over pgx WITHOUT the translating connector wrapper, -// so SQL text reaches Postgres verbatim. It exists for schema DDL: the DDL is -// already native Postgres, and routing it through Translate mangles the -// $$-quoted function bodies and rewrites DDL constructs (e.g. now()). searchPath -// pins the workspace schema exactly as Open does. Callers must use native $n -// placeholders, not ?, since Translate is bypassed. -func OpenRaw(dsn, searchPath string) (*sql.DB, error) { - cfg, err := pgx.ParseConfig(dsn) - if err != nil { - // See Open: pgx does not redact query-param passwords in parse errors. - return nil, ScrubDSNError(dsn, err) - } - if searchPath != "" { - if cfg.RuntimeParams == nil { - cfg.RuntimeParams = map[string]string{} - } - cfg.RuntimeParams["search_path"] = searchPath - } - return sql.OpenDB(stdlib.GetConnector(*cfg)), nil -} - -type connector struct{ base driver.Connector } - -func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { - cn, err := c.base.Connect(ctx) - if err != nil { - return nil, err - } - return &conn{base: cn}, nil -} - -func (c *connector) Driver() driver.Driver { return c.base.Driver() } - -// conn wraps a pgx driver.Conn and translates SQL text at prepare time. -type conn struct{ base driver.Conn } - -func (c *conn) Prepare(query string) (driver.Stmt, error) { - q, err := Translate(query) - if err != nil { - return nil, err - } - base, err := c.base.Prepare(q) - if err != nil { - return nil, err - } - return &stmt{base: base, conn: c.base}, nil -} - -func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { - q, err := Translate(query) - if err != nil { - return nil, err - } - var base driver.Stmt - if p, ok := c.base.(driver.ConnPrepareContext); ok { - base, err = p.PrepareContext(ctx, q) - } else { - base, err = c.base.Prepare(q) - } - if err != nil { - return nil, err - } - return &stmt{base: base, conn: c.base}, nil -} - -func (c *conn) Close() error { return c.base.Close() } - -//nolint:staticcheck // SA1019: driver.Conn requires the deprecated Begin; BeginTx is preferred and forwarded below. -func (c *conn) Begin() (driver.Tx, error) { return c.base.Begin() } - -func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - if b, ok := c.base.(driver.ConnBeginTx); ok { - return b.BeginTx(ctx, opts) - } - return c.base.Begin() //nolint:staticcheck -} - -func (c *conn) ResetSession(ctx context.Context) error { - if rs, ok := c.base.(driver.SessionResetter); ok { - return rs.ResetSession(ctx) - } - return nil -} - -func (c *conn) IsValid() bool { - if v, ok := c.base.(driver.Validator); ok { - return v.IsValid() - } - return true -} - -func (c *conn) Ping(ctx context.Context) error { - if p, ok := c.base.(driver.Pinger); ok { - return p.Ping(ctx) - } - return nil -} - -// stmt wraps a pgx driver.Stmt so bd's MySQL TINYINT(1) discipline survives on -// Postgres: bd binds Go bool arguments to columns that are smallint here (not -// boolean), so CheckNamedValue coerces bool -> 0/1 before binding. Every other -// argument type is forwarded to pgx's own NamedValueChecker so pgx's rich type -// handling (arrays, time, etc.) is preserved. -type stmt struct { - base driver.Stmt - conn driver.Conn -} - -func (s *stmt) CheckNamedValue(nv *driver.NamedValue) error { - if b, ok := nv.Value.(bool); ok { - nv.Value = boolToInt64(b) - return nil - } - if chk, ok := s.conn.(driver.NamedValueChecker); ok { - return chk.CheckNamedValue(nv) - } - return driver.ErrSkip // fall back to database/sql's default conversion -} - -func (s *stmt) Close() error { return s.base.Close() } -func (s *stmt) NumInput() int { return s.base.NumInput() } - -//nolint:staticcheck // driver.Stmt requires the deprecated Exec; ExecContext is preferred and forwarded below. -func (s *stmt) Exec(args []driver.Value) (driver.Result, error) { return s.base.Exec(args) } - -//nolint:staticcheck // driver.Stmt requires the deprecated Query; QueryContext is preferred and forwarded below. -func (s *stmt) Query(args []driver.Value) (driver.Rows, error) { return s.base.Query(args) } - -func (s *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { - if e, ok := s.base.(driver.StmtExecContext); ok { - return e.ExecContext(ctx, args) - } - return s.base.Exec(namedToValues(args)) //nolint:staticcheck -} - -func (s *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { - if q, ok := s.base.(driver.StmtQueryContext); ok { - return q.QueryContext(ctx, args) - } - return s.base.Query(namedToValues(args)) //nolint:staticcheck -} - -func boolToInt64(b bool) int64 { - if b { - return 1 - } - return 0 -} - -func namedToValues(named []driver.NamedValue) []driver.Value { - vals := make([]driver.Value, len(named)) - for i, n := range named { - vals[i] = n.Value - } - return vals -} diff --git a/internal/storage/pgdialect/redact.go b/internal/storage/pgdialect/redact.go deleted file mode 100644 index c415b2f56..000000000 --- a/internal/storage/pgdialect/redact.go +++ /dev/null @@ -1,214 +0,0 @@ -package pgdialect - -import ( - "errors" - "fmt" - "net/url" - "regexp" - "sort" - "strings" - - "github.com/jackc/pgx/v5" -) - -// dsnWSClass is the regexp character class spelling out every byte pgx treats as -// whitespace between libpq keyword/value tokens. Go's regexp \s is [\t\n\f\r ] and -// does NOT include \v (vertical tab); pgx.ParseConfig DOES accept \v as a token -// separator (confirmed: "host=h\vpassword=x" parses host="h", password="x"). Relying -// on \s here would silently fail to match a \v-separated password token at all, -// leaking it in full — so the class is spelled out explicitly instead. -const dsnWSClass = `\t\n\f\r \v` - -// kvPasswordRe matches a libpq keyword/value password token (password= or -// sslpassword=), whose value is either single-quoted or an unquoted run of -// non-whitespace characters that may itself contain backslash-escaped whitespace -// (pgx accepts password=SUPER\ SECRET as the single value `SUPER\ SECRET`, backslash -// retained literally — confirmed against pgx.ParseConfig). -var kvPasswordRe = regexp.MustCompile(`(?i)(^|[` + dsnWSClass + `])(?:password|sslpassword)[` + dsnWSClass + `]*=[` + dsnWSClass + `]*(?:'(?:[^'\\]|\\.)*'|(?:[^` + dsnWSClass + `\\]|\\.)*)`) - -// pwValueRe captures the VALUE of a libpq keyword/value password token: group 1 is -// the single-quoted body, group 2 is an unquoted run of non-whitespace characters, -// including any backslash-escaped whitespace sequences (see kvPasswordRe). -var pwValueRe = regexp.MustCompile(`(?i)(?:^|[` + dsnWSClass + `])(?:password|sslpassword)[` + dsnWSClass + `]*=[` + dsnWSClass + `]*(?:'((?:[^'\\]|\\.)*)'|((?:[^` + dsnWSClass + `\\]|\\.)+))`) - -// RedactPassword returns dsn with the password removed, or an error if a password -// cannot be safely removed. It strips every known password location — URL userinfo, -// URL query params (password, sslpassword), and libpq keyword/value tokens — then -// VERIFIES that no EMBEDDED password survives, failing closed rather than persisting a -// secret in a DSN shape we did not anticipate. The check is embedded-only (HasPassword, -// not pgx.ParseConfig.Password) so an ambient PGPASSWORD/~/.pgpass — which pgx would -// fold in but which is never written to disk — does not falsely block init. -// -// Callers persist the returned string (e.g. to metadata.json) and re-supply the -// password at open time via the credential ladder (see the backend's resolveDSNCredential), -// so nothing operational is lost by never writing the password to disk. The hard -// invariant — a password must never be persisted — is enforced here, not assumed. -func RedactPassword(dsn string) (string, error) { - stripped := stripPasswordBestEffort(dsn) - // It must still parse, or we cannot reason about it — refuse a shape we mangled. - if _, err := pgx.ParseConfig(stripped); err != nil { - return "", fmt.Errorf("cannot verify the connection string is free of a password (unparseable after redaction): %w", err) - } - if HasPassword(stripped) { - return "", fmt.Errorf("refusing to persist a connection string that still carries a password; supply it via BEADS_PG_PASSWORD instead of embedding it in --pg-url") - } - return stripped, nil -} - -// stripPasswordBestEffort removes passwords from the two connection-string shapes -// pgx accepts. The authoritative guarantee is RedactPassword's post-parse check; this -// only has to handle the common forms well enough not to trip that check falsely. -func stripPasswordBestEffort(dsn string) string { - // URL form (scheme://...): strip userinfo password and query password/sslpassword. - if u, err := url.Parse(dsn); err == nil && u.Scheme != "" { - if u.User != nil { - if name := u.User.Username(); name == "" { - u.User = nil - } else { - u.User = url.User(name) - } - } - q := u.Query() - q.Del("password") - q.Del("sslpassword") - u.RawQuery = q.Encode() - return u.String() - } - // libpq keyword/value form (host=... password=... ...): drop the password tokens - // and collapse surrounding whitespace so the remainder still parses. - out := kvPasswordRe.ReplaceAllString(dsn, "$1") - return strings.TrimSpace(strings.Join(strings.Fields(out), " ")) -} - -// ScrubDSNString returns s with every cleartext password embedded in dsn replaced -// by "xxxxx". It is the string-level primitive behind ScrubDSNError, exposed so a -// caller that echoes a DSN somewhere other than an error — a telemetry span, a log -// line — can redact it through the SAME parser-backed extraction (URL userinfo, URL -// query password/sslpassword, and libpq keyword/value tokens) instead of a separate -// hand-rolled scan that misses a form. dsn and s are usually the same string (scrub a -// DSN in place); they differ only when the password leaked into surrounding text, as -// ScrubDSNError passes an error message as s. An empty dsn, or one with no password, -// returns s unchanged. -func ScrubDSNString(dsn, s string) string { - secrets := dsnPasswordValues(dsn) - // Replace the longest secrets first. dsnPasswordValues can return one secret that - // is a byte-for-byte prefix of another (e.g. a userinfo password "foo" and a query - // password "fooACTUAL"); replacing the short one first would turn "fooACTUAL" into - // "xxxxxACTUAL" before the long-secret pass ever runs, leaking the "ACTUAL" tail - // since ReplaceAll can no longer find the (now-mangled) longer substring. - sort.Slice(secrets, func(i, j int) bool { return len(secrets[i]) > len(secrets[j]) }) - for _, secret := range secrets { - s = strings.ReplaceAll(s, secret, "xxxxx") - } - return s -} - -// ScrubDSNError returns a new error whose message has every cleartext password -// embedded in dsn removed. pgx's ParseConfigError redacts only URL userinfo, so a -// `?password=`/`?sslpassword=` URL query param — the shape `bd init` connects with — -// otherwise survives verbatim in the error text and leaks the secret to logs on any -// parse or TLS-config failure. The returned error intentionally does NOT wrap err: -// the pgx error's own Error() still contains the secret, so exposing it through the -// unwrap chain would defeat the redaction. Non-nil in, non-nil out; nil passes through. -func ScrubDSNError(dsn string, err error) error { - if err == nil { - return nil - } - return errors.New(ScrubDSNString(dsn, err.Error())) -} - -// isPasswordKey reports whether k names one of the URL query password params -// (case-insensitively). -func isPasswordKey(k string) bool { - switch strings.ToLower(k) { - case "password", "sslpassword": - return true - } - return false -} - -// dsnPasswordValues returns every cleartext password embedded in dsn — URL userinfo, -// URL query params (password, sslpassword), and libpq keyword/value tokens — in both -// the raw (as-written) and percent-decoded forms, so a value matches however the -// connection string is echoed back. Empty values are skipped, and results are -// de-duplicated. -func dsnPasswordValues(dsn string) []string { - seen := map[string]bool{} - var vals []string - add := func(v string) { - if v == "" || seen[v] { - return - } - seen[v] = true - vals = append(vals, v) - } - if u, err := url.Parse(dsn); err == nil && u.Scheme != "" { - if u.User != nil { - if pw, ok := u.User.Password(); ok { - // url.Parse percent-decodes userinfo, so u.User.Password() is only - // the decoded value. pgx and telemetry sinks echo the DSN verbatim, - // so also scrub the raw as-written form — mirroring the query-param - // branch below, which adds both raw and decoded values. - add(pw) - add(rawURLUserinfoPassword(dsn)) - } - } - // pgx echoes the raw query string back verbatim, so match the raw value; - // also add the decoded value in case a caller logs the parsed form. - for _, pair := range strings.Split(u.RawQuery, "&") { - key, val, ok := strings.Cut(pair, "=") - if !ok { - continue - } - // url.Query() (which pgx.ParseConfig uses) percent-decodes query KEYS as - // well as values, so a query key of pass%77ord or %70assword is a live - // "password" param even though the raw key string doesn't say so. Compare - // both the raw (as-written) key and its unescaped form — ignoring an - // unescape error and falling back to the raw compare, which already ran. - matched := isPasswordKey(key) - if !matched { - if dk, err := url.QueryUnescape(key); err == nil { - matched = isPasswordKey(dk) - } - } - if matched { - add(val) - if dec, err := url.QueryUnescape(val); err == nil { - add(dec) - } - } - } - } - for _, m := range pwValueRe.FindAllStringSubmatch(dsn, -1) { - add(m[1]) // single-quoted body - add(m[2]) // unquoted token - } - return vals -} - -// rawURLUserinfoPassword returns the userinfo password of a URL-form DSN exactly -// as written — still percent-encoded — or "" when there is none. url.Parse decodes -// userinfo, so an encoded secret like SUPER%2ASECRET survives verbatim when a sink -// echoes the raw DSN; extracting it structurally here (rather than re-encoding the -// decoded value) captures the original bytes regardless of encoding form. Mirrors -// the authority split url.Parse performs: password is after the first ':' of the -// userinfo, which is everything before the last '@' of the authority. -func rawURLUserinfoPassword(dsn string) string { - i := strings.Index(dsn, "://") - if i < 0 { - return "" - } - authority := dsn[i+len("://"):] - if j := strings.IndexAny(authority, "/?#"); j >= 0 { - authority = authority[:j] - } - at := strings.LastIndex(authority, "@") - if at < 0 { - return "" - } - _, pw, ok := strings.Cut(authority[:at], ":") - if !ok { - return "" - } - return pw -} diff --git a/internal/storage/pgdialect/redact_error_test.go b/internal/storage/pgdialect/redact_error_test.go deleted file mode 100644 index 4260e6039..000000000 --- a/internal/storage/pgdialect/redact_error_test.go +++ /dev/null @@ -1,188 +0,0 @@ -package pgdialect - -import ( - "database/sql" - "errors" - "strings" - "testing" -) - -// TestScrubDSNError proves an error carrying a DSN can never echo the cleartext -// password back. pgx already redacts URL userinfo and libpq keyword passwords, so -// the URL query-param vectors are the ones that actually leak (see -// TestOpenRedactsPasswordInError); this drives every shape against a worst-case -// error that echoes the raw DSN verbatim. -func TestScrubDSNError(t *testing.T) { - const secret = "SUPERSECRET" - leakForms := []string{secret, "SUPER%2ASECRET", "SUPER*SECRET"} - cases := []struct { - name string - dsn string - }{ - {"url query param", "postgres://u@h:5432/db?password=" + secret}, - {"url query sslpassword", "postgres://u@h:5432/db?sslpassword=" + secret + "&sslmode=require"}, - {"url query param encoded", "postgres://u@h:5432/db?password=SUPER%2ASECRET"}, - {"url userinfo", "postgres://u:" + secret + "@h:5432/db"}, - {"url userinfo encoded", "postgres://u:SUPER%2ASECRET@h:5432/db"}, - {"libpq keyword", "host=h user=u password=" + secret + " dbname=db"}, - {"libpq keyword quoted", "host=h user=u password='" + secret + "' dbname=db"}, - {"libpq sslpassword", "host=h user=u sslpassword=" + secret + " dbname=db"}, - {"url query param percent-encoded key (pass%77ord)", "postgres://u@h:5432/db?pass%77ord=" + secret}, - {"url query sslpassword percent-encoded key (sslpass%77ord)", "postgres://u@h:5432/db?sslpass%77ord=" + secret + "&sslmode=require"}, - {"url query param fully percent-encoded key (%70assword)", "postgres://u@h:5432/db?%70assword=" + secret}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - raw := errors.New("cannot parse `" + tc.dsn + "`: boom") - got := ScrubDSNError(tc.dsn, raw).Error() - for _, form := range leakForms { - if strings.Contains(got, form) { - t.Errorf("scrubbed error still leaks password (%q): %q", form, got) - } - } - // The non-secret failure reason must survive so the error stays useful. - if !strings.Contains(got, "boom") { - t.Errorf("scrubbed error dropped the failure reason: %q", got) - } - }) - } - if ScrubDSNError("postgres://u@h/db?password=x", nil) != nil { - t.Error("ScrubDSNError(dsn, nil) must return nil") - } -} - -// TestScrubDSNString proves the string-level primitive removes every password form -// when a DSN is echoed as plain text (a telemetry span, a log line) rather than an -// error — the same shapes TestScrubDSNError covers, but exercised directly since the -// telemetry scrubber depends on this function, not on ScrubDSNError. -func TestScrubDSNString(t *testing.T) { - const secret = "SUPERSECRET" - leakForms := []string{secret, "SUPER%2ASECRET", "SUPER*SECRET"} - cases := []struct { - name string - dsn string - }{ - {"url userinfo", "postgres://u:" + secret + "@h:5432/db"}, - {"url userinfo encoded", "postgres://u:SUPER%2ASECRET@h:5432/db"}, - {"url query param", "postgres://u@h:5432/db?password=" + secret}, - {"url sslpassword", "postgres://u@h:5432/db?sslpassword=" + secret + "&sslmode=require"}, - {"libpq keyword", "host=h user=u password=" + secret + " dbname=db"}, - {"libpq sslpassword", "host=h user=u sslpassword=" + secret + " dbname=db"}, - {"url query param percent-encoded key (pass%77ord)", "postgres://u@h:5432/db?pass%77ord=" + secret}, - {"url query sslpassword percent-encoded key (sslpass%77ord)", "postgres://u@h:5432/db?sslpass%77ord=" + secret + "&sslmode=require"}, - {"url query param fully percent-encoded key (%70assword)", "postgres://u@h:5432/db?%70assword=" + secret}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got := ScrubDSNString(tc.dsn, tc.dsn) - for _, form := range leakForms { - if strings.Contains(got, form) { - t.Errorf("ScrubDSNString(%q) still leaks password (%q): %q", tc.dsn, form, got) - } - } - if !strings.Contains(got, "xxxxx") { - t.Errorf("ScrubDSNString(%q) did not redact: %q", tc.dsn, got) - } - }) - } - // A DSN with no password, and an empty DSN, must pass the target string through. - if got := ScrubDSNString("postgres://u@h:5432/db", "postgres://u@h:5432/db"); got != "postgres://u@h:5432/db" { - t.Errorf("passwordless DSN altered: %q", got) - } - if got := ScrubDSNString("", "no dsn here"); got != "no dsn here" { - t.Errorf("empty dsn altered s: %q", got) - } -} - -// TestOpenRedactsPasswordInError drives the real pgx parser: a DSN that fails to -// parse while carrying a ?password=/?sslpassword= query param must not leak the -// secret through Open or OpenRaw. connect_timeout=x is what makes ParseConfig fail; -// the query-param password is exactly the vector pgx's own redaction misses. -func TestOpenRedactsPasswordInError(t *testing.T) { - const secret = "SUPERSECRET" - dsns := []string{ - "postgres://u@h:5432/db?password=" + secret + "&connect_timeout=x", - "postgres://u@h:5432/db?sslpassword=" + secret + "&connect_timeout=x", - } - opens := []struct { - name string - open func(string, string) (*sql.DB, error) - }{ - {"Open", Open}, - {"OpenRaw", OpenRaw}, - } - for _, dsn := range dsns { - for _, o := range opens { - t.Run(o.name+" "+dsn, func(t *testing.T) { - db, err := o.open(dsn, "public") - if err == nil { - _ = db.Close() - t.Fatal("expected a parse error, got nil") - } - if strings.Contains(err.Error(), secret) { - t.Errorf("%s leaked password in error: %q", o.name, err.Error()) - } - }) - } - } -} - -// TestScrubDSNStringEscapedWhitespaceValue proves an unquoted libpq keyword/value -// password containing a backslash-escaped space is redacted in full. pgx accepts -// password=SUPER\ SECRET as the single password `SUPER\ SECRET` (the backslash is -// retained literally, the escaped space is part of the value — confirmed against -// pgx.ParseConfig); the pre-fix value regex (a \S+-shaped run) stopped at the -// unescaped whitespace boundary and left the tail " SECRET" behind in plaintext. -func TestScrubDSNStringEscapedWhitespaceValue(t *testing.T) { - dsn := `host=h user=u password=SUPER\ SECRET dbname=db` - got := ScrubDSNString(dsn, dsn) - if strings.Contains(got, "SECRET") { - t.Errorf("ScrubDSNString(%q) leaked escaped-whitespace password tail: %q", dsn, got) - } - if strings.Contains(got, `SUPER\`) { - t.Errorf("ScrubDSNString(%q) leaked escaped-whitespace password head: %q", dsn, got) - } - if !strings.Contains(got, "xxxxx") { - t.Errorf("ScrubDSNString(%q) did not redact: %q", dsn, got) - } - if !strings.Contains(got, "host=h") || !strings.Contains(got, "dbname=db") { - t.Errorf("ScrubDSNString(%q) mangled non-secret structure: %q", dsn, got) - } -} - -// TestScrubDSNStringVerticalTabSeparator proves a password separated from the -// preceding keyword/value token by a vertical tab (\v) is still found and redacted. -// pgx treats \v as inter-token whitespace in libpq keyword/value DSNs (confirmed -// against pgx.ParseConfig: "host=h\vpassword=x" parses host="h", password="x"), but -// Go's regexp \s class is [\t\n\f\r ] and does NOT include \v, so the pre-fix -// password regex — anchored on (^|\s) — never matched at a \v boundary and the -// password leaked entirely. -func TestScrubDSNStringVerticalTabSeparator(t *testing.T) { - const secret = "SUPERSECRET" - dsn := "host=h user=u" + "\v" + "password=" + secret + " dbname=db" - got := ScrubDSNString(dsn, dsn) - if strings.Contains(got, secret) { - t.Errorf("ScrubDSNString(%q) leaked password separated by a vertical tab: %q", dsn, got) - } - if !strings.Contains(got, "xxxxx") { - t.Errorf("ScrubDSNString(%q) did not redact vertical-tab-separated password: %q", dsn, got) - } -} - -// TestScrubDSNStringOverlappingSecretsPrefix proves that when one collected secret is -// a byte-for-byte prefix of another (userinfo password "foo", query password -// "fooACTUAL"), redaction does not leak the longer secret's non-overlapping tail. -// ScrubDSNString applies strings.ReplaceAll sequentially per collected secret; doing -// the short secret ("foo") first turns "fooACTUAL" into "xxxxxACTUAL" before the -// long-secret pass ever runs, so the second pass finds no "fooACTUAL" substring left -// to replace and "ACTUAL" survives in plaintext. -func TestScrubDSNStringOverlappingSecretsPrefix(t *testing.T) { - dsn := "postgres://u:foo@h:5432/db?password=fooACTUAL" - got := ScrubDSNString(dsn, dsn) - if strings.Contains(got, "ACTUAL") { - t.Errorf("ScrubDSNString(%q) leaked the tail of a secret that is a prefix of another secret: %q", dsn, got) - } - if strings.Contains(got, ":foo@") { - t.Errorf("ScrubDSNString(%q) did not redact the shorter overlapping secret: %q", dsn, got) - } -} diff --git a/internal/storage/pgdialect/redact_matrix_test.go b/internal/storage/pgdialect/redact_matrix_test.go deleted file mode 100644 index 22fce592e..000000000 --- a/internal/storage/pgdialect/redact_matrix_test.go +++ /dev/null @@ -1,179 +0,0 @@ -package pgdialect - -import ( - "testing" - - "github.com/jackc/pgx/v5" -) - -// placeholderPW is an obvious non-secret stand-in used wherever a password appears in -// these tests, so nothing here can be mistaken for a leaked real credential. -const placeholderPW = "s3cr3t-PLACEHOLDER" - -// tlsExpect encodes what the redacted DSN's TLS state must be: pgx folds sslmode into -// TLSConfig (sslmode=disable -> nil, sslmode=require -> non-nil), so this is how we -// prove sslmode survived redaction. -type tlsExpect int - -const ( - tlsSkip tlsExpect = iota // don't assert (case sets no sslmode) - tlsMustNil // sslmode=disable -> TLSConfig must be nil - tlsMustSet // sslmode=require -> TLSConfig must be non-nil -) - -// TestRedactPasswordMatrix locks the security invariant across every secret-bearing DSN -// shape pgx accepts. For each case it asserts BOTH: -// -// (a) the persisted string, re-parsed by pgx — the SAME parser that opens the -// connection — carries NO password (prove the secret is gone, not just absent -// from the raw string); and -// (b) every non-secret component (host, port, database, user, sslmode, other query -// params) survives redaction. -// -// A no-password DSN must additionally round-trip byte-for-byte unchanged. -func TestRedactPasswordMatrix(t *testing.T) { - cases := []struct { - name string - dsn string - wantHost string - wantPort uint16 - wantDB string - wantUser string - wantTLS tlsExpect - wantParams map[string]string // RuntimeParams that must be preserved - roundTripSame bool // no-password input must come back unchanged - // skipUser: when the input has no user, pgx defaults cfg.User to the OS user, so - // the parsed user is environment-dependent and not something redaction controls. - skipUser bool - }{ - { - name: "url userinfo password", - dsn: "postgres://bts:" + placeholderPW + "@127.0.0.1:5432/db", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", - }, - { - // The special characters are percent-encoded in userinfo per RFC 3986: - // '@'=%40, ':'=%3A, '/'=%2F. The redactor must still strip it cleanly. - name: "url userinfo password with special chars (url-encoded)", - dsn: "postgres://bts:p%40ss%3Aw%2Frd-" + placeholderPW + "@127.0.0.1:5432/db", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", - }, - { - name: "url query param password + sslmode preserved", - dsn: "postgres://bts@127.0.0.1:5432/db?password=" + placeholderPW + "&sslmode=disable", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", - wantTLS: tlsMustNil, - }, - { - name: "url sslpassword query param + sslmode preserved", - dsn: "postgres://bts@127.0.0.1:5432/db?sslpassword=" + placeholderPW + "&sslmode=require", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", - wantTLS: tlsMustSet, - }, - { - name: "url userinfo password with sslmode and application_name preserved", - dsn: "postgres://bts:" + placeholderPW + "@127.0.0.1:5432/db?sslmode=require&application_name=beads", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", - wantTLS: tlsMustSet, wantParams: map[string]string{"application_name": "beads"}, - }, - { - name: "url password but no user", - dsn: "postgres://:" + placeholderPW + "@127.0.0.1:5432/db", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", skipUser: true, - }, - { - name: "keyword/value password", - dsn: "host=127.0.0.1 port=5432 user=bts password=" + placeholderPW + " dbname=db", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", - }, - { - name: "keyword/value single-quoted password containing spaces", - dsn: "host=127.0.0.1 port=5432 user=bts password='se cret " + placeholderPW + "' dbname=db", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", - }, - { - name: "keyword/value sslpassword + sslmode preserved", - dsn: "host=127.0.0.1 port=5432 user=bts sslpassword=" + placeholderPW + " sslmode=require dbname=db", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", - wantTLS: tlsMustSet, - }, - { - name: "keyword/value password but no user", - dsn: "host=127.0.0.1 port=5432 password=" + placeholderPW + " dbname=db", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", skipUser: true, - }, - { - // Params are in sorted key order because stripPasswordBestEffort re-encodes - // the query via url.Values.Encode(), which sorts keys; the byte-for-byte - // round-trip invariant is asserted against a DSN already in that canonical - // order (application_name < sslmode). - name: "no password (url) round-trips unchanged", - dsn: "postgres://bts@127.0.0.1:5432/db?application_name=beads&sslmode=require", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", - wantTLS: tlsMustSet, wantParams: map[string]string{"application_name": "beads"}, - roundTripSame: true, - }, - { - name: "no password (keyword/value) round-trips unchanged", - dsn: "host=127.0.0.1 port=5432 user=bts dbname=db sslmode=require", - wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", - wantTLS: tlsMustSet, roundTripSame: true, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got, err := RedactPassword(tc.dsn) - if err != nil { - t.Fatalf("RedactPassword(%q) errored: %v", tc.dsn, err) - } - - // (a) The authoritative security check: the same parser pgx uses to open the - // connection must find no password in the persisted string. - cfg, perr := pgx.ParseConfig(got) - if perr != nil { - t.Fatalf("redacted DSN no longer parses: %q: %v", got, perr) - } - if cfg.Password != "" { - t.Fatalf("PASSWORD LEAK: RedactPassword(%q) = %q still carries password %q", tc.dsn, got, cfg.Password) - } - // Belt and suspenders: the embedded-only detector must also agree. - if HasPassword(got) { - t.Fatalf("PASSWORD LEAK: HasPassword still true for redacted %q", got) - } - - // (b) Non-secret components must survive. - if cfg.Host != tc.wantHost { - t.Errorf("host = %q, want %q", cfg.Host, tc.wantHost) - } - if cfg.Port != tc.wantPort { - t.Errorf("port = %d, want %d", cfg.Port, tc.wantPort) - } - if cfg.Database != tc.wantDB { - t.Errorf("database = %q, want %q", cfg.Database, tc.wantDB) - } - if !tc.skipUser && cfg.User != tc.wantUser { - t.Errorf("user = %q, want %q", cfg.User, tc.wantUser) - } - switch tc.wantTLS { - case tlsMustNil: - if cfg.TLSConfig != nil { - t.Errorf("sslmode not preserved: TLSConfig = %v, want nil (sslmode=disable)", cfg.TLSConfig) - } - case tlsMustSet: - if cfg.TLSConfig == nil { - t.Errorf("sslmode not preserved: TLSConfig = nil, want non-nil (sslmode=require)") - } - } - for k, want := range tc.wantParams { - if got := cfg.RuntimeParams[k]; got != want { - t.Errorf("param %q = %q, want %q (must be preserved)", k, got, want) - } - } - - if tc.roundTripSame && got != tc.dsn { - t.Errorf("no-password DSN must round-trip unchanged:\n in = %q\n out = %q", tc.dsn, got) - } - }) - } -} diff --git a/internal/storage/pgdialect/redact_test.go b/internal/storage/pgdialect/redact_test.go deleted file mode 100644 index bc07f6a05..000000000 --- a/internal/storage/pgdialect/redact_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package pgdialect - -import ( - "testing" - - "github.com/jackc/pgx/v5" -) - -// TestRedactPassword encodes the hard invariant: for every DSN shape pgx accepts, -// RedactPassword's output must carry NO password (verified by re-parsing with pgx), -// while the rest of the connection string survives. These cases include the two -// vectors the red-team confirmed leaked: a query-param password and a libpq -// keyword/value password. -func TestRedactPassword(t *testing.T) { - cases := []struct { - name string - dsn string - }{ - {"url userinfo", "postgres://bts:secret@127.0.0.1:5432/db"}, - {"url query param", "postgres://bts@127.0.0.1:5432/db?password=secret"}, - {"url query param + other", "postgres://bts@127.0.0.1:5432/db?password=secret&sslmode=disable"}, - {"url sslpassword", "postgres://bts@127.0.0.1:5432/db?sslpassword=secret&sslmode=require"}, - {"libpq kv", "host=127.0.0.1 port=5432 user=bts password=secret dbname=db"}, - {"libpq kv quoted", "host=127.0.0.1 user=bts password='se cret' dbname=db"}, - {"libpq kv sslpassword", "host=127.0.0.1 user=bts sslpassword=secret dbname=db"}, - {"no password (url)", "postgres://bts@127.0.0.1:5432/db"}, - {"no password (kv)", "host=127.0.0.1 user=bts dbname=db"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got, err := RedactPassword(tc.dsn) - if err != nil { - t.Fatalf("RedactPassword(%q) errored: %v", tc.dsn, err) - } - // The authoritative check: pgx — the same parser that opens the - // connection — must see no password in the persisted string. - cfg, perr := pgx.ParseConfig(got) - if perr != nil { - t.Fatalf("redacted DSN no longer parses: %q: %v", got, perr) - } - if cfg.Password != "" { - t.Fatalf("PASSWORD LEAK: RedactPassword(%q) = %q still carries password %q", tc.dsn, got, cfg.Password) - } - }) - } -} - -// TestRedactPasswordFailsClosed: if the input still has a password that the -// best-effort strip missed, RedactPassword must ERROR rather than return a leaky -// string. We simulate an "unanticipated shape" by asserting the post-parse guard is -// what enforces this — a KV password with an exotic separator that the regex could -// miss must never silently pass through with a password. -func TestRedactPasswordNeverReturnsASecret(t *testing.T) { - // Sanity: a plain non-DSN string must not be returned as a "clean" DSN. - if _, err := RedactPassword("::::not a url"); err == nil { - t.Fatal("expected error for an unparseable connection string, got nil") - } -} diff --git a/internal/storage/pgdialect/translate.go b/internal/storage/pgdialect/translate.go deleted file mode 100644 index fa462bdc3..000000000 --- a/internal/storage/pgdialect/translate.go +++ /dev/null @@ -1,778 +0,0 @@ -// Package pgdialect translates the MySQL-dialect SQL that bd's shared storage -// layer (internal/storage/sqlbuild and internal/storage/issueops) emits into -// PostgreSQL, so that same layer can run unchanged over a pgx-backed -// database/sql connection. -// -// It deliberately handles only the constructs bd actually emits on the storage -// path — a finite, enumerable set (see corpus_test.go). Anything it does not -// recognize passes through untouched and surfaces as a loud Postgres syntax -// error at PREPARE time, never a silent mistranslation. That "fail loud" -// property is the whole safety argument: a divergence is a build failure, not a -// subtly wrong row set. -package pgdialect - -import ( - "fmt" - "strings" -) - -// Translate rewrites a single MySQL statement into its Postgres equivalent. -// -// Pass order is deliberate. Function-shaped rewrites (IF, VALUES, CONCAT, -// LOCATE, JSON) run first so the ON DUPLICATE KEY body is already Postgres by -// the time it is wrapped into ON CONFLICT. Placeholder numbering runs last, so -// every earlier pass can move text freely; none of them introduce a literal -// `?`, so left-to-right numbering stays correct. -func Translate(sql string) (string, error) { - out := sql - // CONCAT/LOCATE are unambiguous functions anywhere they appear. IF() and - // VALUES() are NOT translated globally: VALUES is also the INSERT keyword, - // and both only appear as functions inside an ON DUPLICATE KEY UPDATE body, - // so they are translated there (rewriteOnDuplicateKey). - out = rewriteFuncCalls(out, "CONCAT", concatArgs) // CONCAT(a,b) -> (a || b) - out = rewriteFuncCalls(out, "LOCATE", locateArgs) // LOCATE(n,h) -> POSITION(n IN h) - out = rewriteFuncCalls(out, "INSTR", instrArgs) // INSTR(h,n) -> POSITION(n IN h) (args reversed vs LOCATE) - out = rewriteFuncCalls(out, "DATE_FORMAT", dateFormatArgs) // DATE_FORMAT(t,'%Y-%m-%dT%H:%i:%sZ') -> to_char(t,'YYYY-MM-DD"T"HH24:MI:SS"Z"') - out = rewriteFuncCalls(out, "JSON_OBJECT", jsonObjectArgs) // JSON_OBJECT(k,v,...) -> jsonb_build_object(k,v,...) - out = rewriteCastChar(out) // CAST(x AS CHAR[(n)]) -> (x)::text - // CURRENT_TIMESTAMP (not now()) so the replacement contains no token that a - // later pass would re-match. AT TIME ZONE 'utc' yields naive-UTC to match - // bd's Dolt storage discipline. - out = replaceIdentAll(out, "UTC_TIMESTAMP()", "(CURRENT_TIMESTAMP AT TIME ZONE 'utc')") - out = replaceIdentAll(out, "NOW()", "(CURRENT_TIMESTAMP AT TIME ZONE 'utc')") - out = rewriteLiteralJSONUnquote(out) - out = rewriteDynamicJSONPath(out) - out = rewriteLabelOrderBy(out) - out = rewriteInsertIgnore(out) - out = rewriteReplaceInto(out) - out = rewriteOnDuplicateKey(out) - out = rewriteUpdateSetTargets(out) - out = rewriteBackticks(out) - out = numberPlaceholders(out) - return out, nil -} - -// codeMask returns a per-byte bitmap: mask[i] is true iff byte i lies outside -// any string literal or comment. bd uses single-quoted strings (” escapes a -// quote), `--` line comments, and `/* */` blocks; it never double-quotes -// strings (those are identifiers in Postgres/ANSI mode), so a double quote is -// always code. Every text rewrite consults this so it never touches quoted data -// or commented-out SQL. -func codeMask(sql string) []bool { - b := []byte(sql) - m := make([]bool, len(b)) - var inStr, inLine, inBlock bool - for i := 0; i < len(b); i++ { - c := b[i] - switch { - case inLine: - if c == '\n' { - inLine = false - } - case inBlock: - if c == '*' && i+1 < len(b) && b[i+1] == '/' { - i++ // consume '/' - inBlock = false - } - case inStr: - if c == '\'' { - if i+1 < len(b) && b[i+1] == '\'' { - i++ // escaped quote: consume both - } else { - inStr = false - } - } - default: - switch { - case c == '\'': - inStr = true - case c == '-' && i+1 < len(b) && b[i+1] == '-': - i++ - inLine = true - case c == '/' && i+1 < len(b) && b[i+1] == '*': - i++ - inBlock = true - default: - m[i] = true - } - } - } - return m -} - -func isIdentByte(c byte) bool { - return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') -} - -// numberPlaceholders replaces each code-region `?` with $1, $2, … in order. -func numberPlaceholders(sql string) string { - m := codeMask(sql) - var b strings.Builder - n := 0 - for i := 0; i < len(sql); i++ { - if m[i] && sql[i] == '?' { - n++ - fmt.Fprintf(&b, "$%d", n) - continue - } - b.WriteByte(sql[i]) - } - return b.String() -} - -// rewriteBackticks turns `ident` into "ident" in code regions. bd only uses -// backticks around reserved-word column names (e.g. `key`). -func rewriteBackticks(sql string) string { - m := codeMask(sql) - out := []byte(sql) - for i := range out { - if m[i] && out[i] == '`' { - out[i] = '"' - } - } - return string(out) -} - -// replaceIdentAll replaces every code-region occurrence of a literal token -// (case-insensitive) with repl. Used for parameterless functions like -// UTC_TIMESTAMP() and NOW(). -func replaceIdentAll(sql, token, repl string) string { - lower := strings.ToLower(token) - m := codeMask(sql) - var b strings.Builder - // Single forward pass: never re-scans the inserted replacement, so a repl - // that contains the token (NOW() -> "(now() …)") can't loop. - for i := 0; i < len(sql); { - if m[i] && i+len(token) <= len(sql) && strings.ToLower(sql[i:i+len(token)]) == lower { - b.WriteString(repl) - i += len(token) - continue - } - b.WriteByte(sql[i]) - i++ - } - return b.String() -} - -// rewriteFuncCalls finds NAME(args…) in code regions (balanced parens, -// string-aware) and replaces each with rebuild(args). It recurses on each -// argument first, so nested calls (LOCATE(CONCAT(…), x)) translate correctly. -func rewriteFuncCalls(sql, name string, rebuild func(args []string) string) string { - lower := strings.ToLower(name) - searchFrom := 0 - for { - idx := findCallStart(sql, lower, searchFrom) - if idx < 0 { - return sql - } - args, end, ok := splitCallArgs(sql, idx+len(name)) - if !ok { - searchFrom = idx + len(name) // not a call; skip past this token - continue - } - for i := range args { - args[i] = rewriteFuncCalls(strings.TrimSpace(args[i]), name, rebuild) - } - rebuilt := rebuild(args) - sql = sql[:idx] + rebuilt + sql[end:] - // Advance past the rebuilt region — nested same-name calls were already - // handled by the recursion on args, so we never need to revisit it. This - // guarantees forward progress (termination) even if a rebuild happened to - // contain the function name. - searchFrom = idx + len(rebuilt) - } -} - -// findCallStart returns the index of the next code-region, word-boundaried -// occurrence of name (case-insensitive) that is immediately followed by `(` -// (optionally after spaces), at or after `from`; or -1. -func findCallStart(sql, lowerName string, from int) int { - m := codeMask(sql) - for i := from; i+len(lowerName) <= len(sql); i++ { - if !m[i] { - continue - } - if strings.ToLower(sql[i:i+len(lowerName)]) != lowerName { - continue - } - if i > 0 && isIdentByte(sql[i-1]) { - continue // not a word boundary - } - j := i + len(lowerName) - for j < len(sql) && (sql[j] == ' ' || sql[j] == '\t' || sql[j] == '\n') { - j++ - } - if j < len(sql) && sql[j] == '(' { - return i - } - } - return -1 -} - -// splitCallArgs, given the index just after a function name, finds the opening -// paren, walks to the matching close (string-aware), and returns the top-level -// comma-separated arguments plus the index just past the close paren. -func splitCallArgs(sql string, afterName int) (args []string, end int, ok bool) { - m := codeMask(sql) - i := afterName - for i < len(sql) && sql[i] != '(' { - if sql[i] != ' ' && sql[i] != '\t' && sql[i] != '\n' { - return nil, 0, false - } - i++ - } - if i >= len(sql) || sql[i] != '(' { - return nil, 0, false - } - depth := 0 - start := i + 1 - for ; i < len(sql); i++ { - if !m[i] { - continue - } - switch sql[i] { - case '(': - depth++ - case ')': - depth-- - if depth == 0 { - args = append(args, sql[start:i]) - return args, i + 1, true - } - case ',': - if depth == 1 { - args = append(args, sql[start:i]) - start = i + 1 - } - } - } - return nil, 0, false -} - -func ifArgs(args []string) string { - if len(args) != 3 { - return "IF(" + strings.Join(args, ", ") + ")" // leave loud - } - return "CASE WHEN " + strings.TrimSpace(args[0]) + " THEN " + strings.TrimSpace(args[1]) + - " ELSE " + strings.TrimSpace(args[2]) + " END" -} - -func valuesArgs(args []string) string { - if len(args) != 1 { - return "VALUES(" + strings.Join(args, ", ") + ")" - } - return "EXCLUDED." + strings.TrimSpace(args[0]) -} - -func concatArgs(args []string) string { - for i := range args { - args[i] = strings.TrimSpace(args[i]) - } - return "(" + strings.Join(args, " || ") + ")" -} - -func locateArgs(args []string) string { - if len(args) != 2 { - return "LOCATE(" + strings.Join(args, ", ") + ")" // 3-arg form not on bd's path; leave loud - } - return "POSITION(" + strings.TrimSpace(args[0]) + " IN " + strings.TrimSpace(args[1]) + ")" -} - -func instrArgs(args []string) string { - if len(args) != 2 { - return "INSTR(" + strings.Join(args, ", ") + ")" // leave loud - } - // POSITION reverses INSTR's arg order. Placeholder numbering runs later and - // left-to-right, so reordering two args that BOTH carry a `?` would swap their - // $-numbers relative to the caller's bound-arg order — a silent wrong-parameter - // mistranslation. bd only ever emits INSTR(col, ?), so guard the ambiguous case - // by leaving it loud to fail at PREPARE rather than swap bindings. - if strings.Contains(args[0], "?") && strings.Contains(args[1], "?") { - return "INSTR(" + strings.Join(args, ", ") + ")" - } - // MySQL INSTR(haystack, needle) — note the reversed order vs LOCATE. - return "POSITION(" + strings.TrimSpace(args[1]) + " IN " + strings.TrimSpace(args[0]) + ")" -} - -// mysqlDateFormatISO8601 is the ONLY DATE_FORMAT pattern bd emits (counts.go's -// DepJSONObject renders created_at as RFC3339). Any other format literal is -// left untranslated so it fails loudly at PREPARE rather than silently -// producing a differently-shaped timestamp. -const ( - mysqlDateFormatISO8601 = `'%Y-%m-%dT%H:%i:%sZ'` - pgDateFormatISO8601 = `'YYYY-MM-DD"T"HH24:MI:SS"Z"'` -) - -func dateFormatArgs(args []string) string { - if len(args) != 2 || strings.TrimSpace(args[1]) != mysqlDateFormatISO8601 { - return "DATE_FORMAT(" + strings.Join(args, ", ") + ")" // unknown format: leave loud - } - return "to_char(" + strings.TrimSpace(args[0]) + ", " + pgDateFormatISO8601 + ")" -} - -// jsonObjectArgs maps MySQL's comma-form JSON_OBJECT(k1, v1, k2, v2, …) to -// Postgres jsonb_build_object with the same alternating key/value args. An odd -// (or empty) arg count is not a valid object literal, so it is left loud. -func jsonObjectArgs(args []string) string { - if len(args) == 0 || len(args)%2 != 0 { - return "JSON_OBJECT(" + strings.Join(args, ", ") + ")" // leave loud - } - for i := range args { - args[i] = strings.TrimSpace(args[i]) - } - return "jsonb_build_object(" + strings.Join(args, ", ") + ")" -} - -// rewriteInsertIgnore turns `INSERT IGNORE INTO …` into `INSERT INTO … ON -// CONFLICT DO NOTHING`. bd uses it only for idempotent child-table inserts -// (labels, deps, events, comments) with no trailing clause after VALUES/SELECT. -func rewriteInsertIgnore(sql string) string { - const ins = "insert ignore" // bd emits a single space between the keywords - idx := indexCodeFold(sql, ins) - if idx < 0 { - return sql - } - out := sql[:idx] + "INSERT" + sql[idx+len(ins):] - if !containsCodeFold(out, "ON CONFLICT") { - out = strings.TrimRight(out, " \n\t") + " ON CONFLICT DO NOTHING" - } - return out -} - -// rewriteLabelOrderBy makes label sorting explicit on Postgres. The shared -// query text stays raw (`ORDER BY label` / `ORDER BY issue_id, label`) so the -// Dolt reference path remains unchanged; the SQL backends translate it to a -// code-point collation here. -func rewriteLabelOrderBy(sql string) string { - sql = replaceIdentAll(sql, "ORDER BY issue_id, label", `ORDER BY issue_id, label COLLATE "C"`) - return replaceIdentAll(sql, "ORDER BY label", `ORDER BY label COLLATE "C"`) -} - -// rewriteReplaceInto converts MySQL `REPLACE INTO t (c1, c2, …) VALUES …` into -// `INSERT INTO t (c1, c2, …) VALUES … ON CONFLICT (c1) DO UPDATE SET c2 = -// EXCLUDED.c2, …`. bd only uses REPLACE on key/value tables (config, metadata, -// local_metadata) where the first column is the primary key, so treating column -// 1 as the conflict target and overwriting the rest is faithful. The column -// tokens keep their backticks here; the later rewriteBackticks pass quotes them. -func rewriteReplaceInto(sql string) string { - const rep = "replace into" - idx := indexCodeFold(sql, rep) - if idx < 0 { - return sql - } - out := sql[:idx] + "INSERT INTO" + sql[idx+len(rep):] - paren := strings.Index(out[idx:], "(") - if paren < 0 { - return out // no column list — leave loud - } - cols, _, ok := splitCallArgs(out, idx+paren) - if !ok || len(cols) == 0 { - return out - } - target := strings.TrimSpace(cols[0]) - sets := make([]string, 0, len(cols)-1) - for _, c := range cols[1:] { - c = strings.TrimSpace(c) - sets = append(sets, c+" = EXCLUDED."+c) - } - tail := " ON CONFLICT (" + target + ") DO NOTHING" - if len(sets) > 0 { - tail = " ON CONFLICT (" + target + ") DO UPDATE SET " + strings.Join(sets, ", ") - } - return strings.TrimRight(out, " \n\t") + tail -} - -func skipWS(sql string, i int) int { - for i < len(sql) && (sql[i] == ' ' || sql[i] == '\t' || sql[i] == '\n' || sql[i] == '\r') { - i++ - } - return i -} - -// wordAtFold is wordAt with case-insensitive matching (for SQL keywords). -func wordAtFold(sql string, i int, word string) bool { - if i+len(word) > len(sql) || !strings.EqualFold(sql[i:i+len(word)], word) { - return false - } - if i > 0 && isIdentByte(sql[i-1]) { - return false - } - if i+len(word) < len(sql) && isIdentByte(sql[i+len(word)]) { - return false - } - return true -} - -func isIdentWord(s string) bool { - if s == "" { - return false - } - for i := 0; i < len(s); i++ { - if !isIdentByte(s[i]) { - return false - } - } - return true -} - -// rewriteUpdateSetTargets strips the table-alias qualifier from UPDATE SET -// targets: MySQL accepts `UPDATE t a SET a.col = …`, but Postgres requires the -// SET target to be unqualified (`SET col = …`) — it reads `a` as a column -// otherwise. Only the assignment target (left of the top-level `=`) is -// unqualified; alias references on the right-hand side are left intact. -func rewriteUpdateSetTargets(sql string) string { - m := codeMask(sql) - start := skipWS(sql, 0) - if !wordAtFold(sql, start, "update") { - return sql - } - setPos := scanKeyword(sql, m, start, "set") - if setPos < 0 { - return sql - } - clauseStart := setPos + len("set") - clauseEnd := scanKeyword(sql, m, clauseStart, "where") - if clauseEnd < 0 { - if r := scanKeyword(sql, m, clauseStart, "returning"); r >= 0 { - clauseEnd = r - } else { - clauseEnd = len(sql) - } - } - clause := sql[clauseStart:clauseEnd] - cm := codeMask(clause) - var out strings.Builder - depth, segStart := 0, 0 - for j := 0; j < len(clause); j++ { - if cm[j] { - switch clause[j] { - case '(': - depth++ - case ')': - depth-- - case ',': - if depth == 0 { - out.WriteString(unqualifySetTarget(clause[segStart:j])) - out.WriteByte(',') - segStart = j + 1 - } - } - } - } - out.WriteString(unqualifySetTarget(clause[segStart:])) - return sql[:clauseStart] + out.String() + sql[clauseEnd:] -} - -// scanKeyword returns the index of the next depth-0, code-region occurrence of -// keyword (word-boundaried, case-insensitive) at or after from, or -1. -func scanKeyword(sql string, m []bool, from int, keyword string) int { - depth := 0 - for j := from; j < len(sql); j++ { - if !m[j] { - continue - } - switch sql[j] { - case '(': - depth++ - case ')': - depth-- - } - if depth == 0 && wordAtFold(sql, j, keyword) { - return j - } - } - return -1 -} - -func unqualifySetTarget(seg string) string { - cm := codeMask(seg) - depth, eq := 0, -1 - for j := 0; j < len(seg) && eq < 0; j++ { - if !cm[j] { - continue - } - switch seg[j] { - case '(': - depth++ - case ')': - depth-- - case '=': - if depth == 0 { - eq = j - } - } - } - if eq < 0 { - return seg - } - lhs := seg[:eq] - trimmed := strings.TrimSpace(lhs) - dot := strings.IndexByte(trimmed, '.') - if dot <= 0 || !isIdentWord(trimmed[:dot]) || !isIdentWord(trimmed[dot+1:]) { - return seg - } - lead := lhs[:len(lhs)-len(strings.TrimLeft(lhs, " \t\r\n"))] - return lead + trimmed[dot+1:] + " " + seg[eq:] -} - -// onConflictTargets maps a table to the columns Postgres needs as the ON -// CONFLICT arbiter (the primary/unique key MySQL's ON DUPLICATE KEY infers). -// Only tables that use a DO UPDATE upsert need an entry; DO NOTHING needs none. -var onConflictTargets = map[string]string{ - "issues": "id", - "wisps": "id", - "issue_counter": "prefix", - "child_counters": "parent_id", - "wisp_child_counters": "parent_id", - "repo_mtimes": "repo_path", - "federation_peers": "name", - "leases": "issue_id", -} - -// rewriteOnDuplicateKey converts `… ON DUPLICATE KEY UPDATE ` into -// Postgres `… ON CONFLICT () DO UPDATE SET `, or `ON CONFLICT DO -// NOTHING` when the body is an idempotent self-assignment (`col = col`). The ON -// DUPLICATE clause is always last in the statement, so the body runs to the end. -func rewriteOnDuplicateKey(sql string) string { - const odk = "on duplicate key update" - i := indexCodeFold(sql, odk) - if i < 0 { - return sql - } - head := strings.TrimRight(sql[:i], " \n\t") - body := strings.TrimSpace(sql[i+len(odk):]) - if isSelfAssign(body) { - return head + " ON CONFLICT DO NOTHING" - } - // Inside the upsert body, VALUES(col) means the proposed row value - // (Postgres EXCLUDED.col) and IF(c,a,b) is the ternary (CASE). Translate them - // here, where they can't be confused with the INSERT ... VALUES keyword. - body = rewriteFuncCalls(body, "IF", ifArgs) - body = rewriteFuncCalls(body, "VALUES", valuesArgs) - target := onConflictTargets[insertTable(sql)] - if target == "" { - // Unknown upsert target: emit a form that fails PREPARE loudly rather - // than guessing an arbiter (surfaced by the corpus gate). - return head + " ON CONFLICT () DO UPDATE SET " + body - } - return head + " ON CONFLICT (" + target + ") DO UPDATE SET " + body -} - -// isSelfAssign reports whether body is a single `x = x` assignment. -func isSelfAssign(body string) bool { - parts := strings.SplitN(body, "=", 2) - if len(parts) != 2 { - return false - } - l := strings.TrimSpace(parts[0]) - r := strings.TrimSpace(strings.TrimRight(parts[1], " \n\t")) - return l != "" && l == r && !strings.ContainsAny(l, " (),") -} - -// insertTable extracts the target table from a leading `INSERT [IGNORE] INTO -//
`. The table is concrete at runtime (fmt.Sprintf filled it in). -func insertTable(sql string) string { - toks := strings.Fields(sql) - for i := 0; i+1 < len(toks); i++ { - if strings.EqualFold(toks[i], "INTO") { - t := strings.Trim(toks[i+1], "`\"") - if p := strings.IndexAny(t, "("); p >= 0 { - t = t[:p] - } - return t - } - } - return "" -} - -// rewriteLiteralJSONUnquote handles the fixed-path form -// JSON_UNQUOTE(JSON_EXTRACT(, '$.')) (the waits-for gate check), -// rewriting it to ( #>> '{}'). Only the single-level literal-path -// variant is handled; dynamic ?-path metadata filters translate separately. -func rewriteLiteralJSONUnquote(sql string) string { - const marker = "json_unquote(json_extract(" - for { - i := indexCodeFold(sql, marker) - if i < 0 { - return sql - } - open := i + len("JSON_UNQUOTE(") + len("JSON_EXTRACT") - args, extractEnd, ok := splitCallArgs(sql, open) - if !ok || len(args) != 2 || extractEnd >= len(sql) || sql[extractEnd] != ')' { - return sql // malformed / not the literal form: leave loud - } - key := jsonPathToKey(strings.TrimSpace(args[1])) - if key == "" { - return sql // dynamic path: leave for the param translator - } - repl := "(" + strings.TrimSpace(args[0]) + " #>> '{" + key + "}')" - sql = sql[:i] + repl + sql[extractEnd+1:] - } -} - -// rewriteDynamicJSONPath translates the dynamic ?-path metadata filters -// sqlbuild emits (filter.go): the has-key form JSON_EXTRACT(metadata, ?) IS NOT -// NULL and the equality form JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) = ?. The -// MySQL path argument (e.g. '$.sprint' or '$."gc.routed_to"') is a bound -// parameter, so the bare-key conversion happens at runtime via the -// bd_mysql_jsonkey SQL function (defined in the Postgres schema): -// -// JSON_UNQUOTE(JSON_EXTRACT(, ?)) -> ( ->> bd_mysql_jsonkey(?)) -// JSON_EXTRACT(, ?) -> ( -> bd_mysql_jsonkey(?)) -// -// The unquote form is rewritten first so its inner JSON_EXTRACT is consumed -// before the bare-extract pass runs. Literal-path forms (a quoted '$.key' -// argument) are left untouched here; rewriteLiteralJSONUnquote already handled -// them and any bare literal JSON_EXTRACT is left loud. -func rewriteDynamicJSONPath(sql string) string { - sql = rewriteDynamicJSONUnquote(sql) - sql = rewriteDynamicJSONExtract(sql) - return sql -} - -func rewriteDynamicJSONUnquote(sql string) string { - const marker = "json_unquote(json_extract(" - searchFrom := 0 - for { - i := indexCodeFoldFrom(sql, marker, searchFrom) - if i < 0 { - return sql - } - open := i + len("JSON_UNQUOTE(") + len("JSON_EXTRACT") - args, extractEnd, ok := splitCallArgs(sql, open) - if !ok || len(args) != 2 || extractEnd >= len(sql) || sql[extractEnd] != ')' { - searchFrom = i + len(marker) // malformed / not this form: skip - continue - } - if strings.TrimSpace(args[1]) != "?" { - searchFrom = i + len(marker) // literal path: leave for the literal pass - continue - } - repl := "(" + strings.TrimSpace(args[0]) + " ->> bd_mysql_jsonkey(?))" - sql = sql[:i] + repl + sql[extractEnd+1:] // +1 consumes the outer JSON_UNQUOTE ')' - searchFrom = i + len(repl) - } -} - -func rewriteDynamicJSONExtract(sql string) string { - searchFrom := 0 - for { - idx := findCallStart(sql, "json_extract", searchFrom) - if idx < 0 { - return sql - } - args, end, ok := splitCallArgs(sql, idx+len("JSON_EXTRACT")) - if !ok || len(args) != 2 || strings.TrimSpace(args[1]) != "?" { - searchFrom = idx + len("JSON_EXTRACT") // literal / malformed path: leave loud - continue - } - repl := "(" + strings.TrimSpace(args[0]) + " -> bd_mysql_jsonkey(?))" - sql = sql[:idx] + repl + sql[end:] - searchFrom = idx + len(repl) - } -} - -// rewriteCastChar rewrites CAST( AS CHAR) and CAST( AS CHAR(n)) into -// ()::text. bd emits CAST(metadata AS CHAR) to serialize a JSON column to -// its text form (counts.go DepJSONObject). This rewrite is MANDATORY, not -// cosmetic: Postgres reads a bare CHAR as char(1), so CAST(jsonb AS CHAR) -// prepares silently and truncates the value to a single character. -func rewriteCastChar(sql string) string { - searchFrom := 0 - for { - idx := findCallStart(sql, "cast", searchFrom) - if idx < 0 { - return sql - } - args, end, ok := splitCallArgs(sql, idx+len("CAST")) - if !ok || len(args) != 1 { - searchFrom = idx + len("CAST") - continue - } - expr, isChar := splitCastAsChar(args[0]) - if !isChar { - searchFrom = idx + len("CAST") // not AS CHAR (e.g. AS SIGNED): leave loud - continue - } - repl := "(" + strings.TrimSpace(expr) + ")::text" - sql = sql[:idx] + repl + sql[end:] - searchFrom = idx + len(repl) - } -} - -// splitCastAsChar reports whether inner has the shape " AS CHAR" or -// " AS CHAR(n)" and, if so, returns . The AS is matched at paren -// depth 0 in a code region, so an AS nested inside (e.g. a nested CAST) -// is ignored. -func splitCastAsChar(inner string) (expr string, ok bool) { - m := codeMask(inner) - depth := 0 - for i := 0; i < len(inner); i++ { - if !m[i] { - continue - } - switch inner[i] { - case '(': - depth++ - case ')': - depth-- - } - if depth == 0 && wordAtFold(inner, i, "as") { - if isCharType(strings.TrimSpace(inner[i+len("as"):])) { - return inner[:i], true - } - return "", false - } - } - return "", false -} - -// isCharType reports whether tail is a CHAR type token: "CHAR" or "CHAR(n)". -func isCharType(tail string) bool { - if !wordAtFold(tail, 0, "char") { - return false - } - rest := strings.TrimSpace(tail[len("char"):]) - if rest == "" { - return true // CHAR - } - return rest[0] == '(' && rest[len(rest)-1] == ')' // CHAR(n) -} - -// jsonPathToKey converts a simple literal MySQL JSON path '$.key' into the bare -// key, or "" if it is not a simple single-level literal. -func jsonPathToKey(lit string) string { - if len(lit) < 4 || lit[0] != '\'' || lit[len(lit)-1] != '\'' { - return "" - } - p := lit[1 : len(lit)-1] - if !strings.HasPrefix(p, "$.") { - return "" - } - k := strings.Trim(p[2:], `"`) - if k == "" || strings.ContainsAny(k, ".[]*") { - return "" - } - return k -} - -func indexCodeFold(sql, sub string) int { return indexCodeFoldFrom(sql, sub, 0) } - -// indexCodeFoldFrom is indexCodeFold starting the scan at `from`. Rewrites that -// may legitimately leave a match in place (a literal path they must skip) use -// this to advance past it instead of looping forever. -func indexCodeFoldFrom(sql, sub string, from int) int { - if from < 0 { - from = 0 - } - m := codeMask(sql) - lower := strings.ToLower(sub) - for i := from; i+len(sub) <= len(sql); i++ { - if m[i] && strings.ToLower(sql[i:i+len(sub)]) == lower { - return i - } - } - return -1 -} - -func containsCodeFold(sql, sub string) bool { return indexCodeFold(sql, sub) >= 0 } diff --git a/internal/storage/pgdialect/translate_test.go b/internal/storage/pgdialect/translate_test.go deleted file mode 100644 index d201ac0af..000000000 --- a/internal/storage/pgdialect/translate_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package pgdialect - -import "testing" - -func TestTranslate(t *testing.T) { - cases := []struct { - name string - in string - want string - }{ - { - name: "placeholders numbered", - in: `SELECT * FROM issues WHERE status = ? AND priority = ?`, - want: `SELECT * FROM issues WHERE status = $1 AND priority = $2`, - }, - { - name: "placeholder inside string is left alone", - in: `SELECT '?', assignee FROM issues WHERE id = ?`, - want: `SELECT '?', assignee FROM issues WHERE id = $1`, - }, - { - name: "backtick reserved-word identifier", - in: "SELECT value FROM config WHERE `key` = ?", - want: `SELECT value FROM config WHERE "key" = $1`, - }, - { - name: "INSERT IGNORE -> ON CONFLICT DO NOTHING", - in: `INSERT IGNORE INTO labels (issue_id, label) VALUES (?, ?)`, - want: `INSERT INTO labels (issue_id, label) VALUES ($1, $2) ON CONFLICT DO NOTHING`, - }, - { - name: "ON DUPLICATE KEY type=type -> ON CONFLICT DO NOTHING", - in: `INSERT INTO dependencies (id, issue_id, depends_on_issue_id, type, created_by, created_at) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE type = type`, - want: `INSERT INTO dependencies (id, issue_id, depends_on_issue_id, type, created_by, created_at) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING`, - }, - { - name: "issue upsert VALUES(col) -> EXCLUDED.col with (id) target", - in: `INSERT INTO issues (id, title) VALUES (?, ?) ON DUPLICATE KEY UPDATE title = VALUES(title)`, - want: `INSERT INTO issues (id, title) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET title = EXCLUDED.title`, - }, - { - name: "stale-guard IF(VALUES...)-> CASE WHEN EXCLUDED", - in: `INSERT INTO issues (id, title, updated_at) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE title = IF(VALUES(updated_at) > updated_at, VALUES(title), title)`, - want: `INSERT INTO issues (id, title, updated_at) VALUES ($1, $2, $3) ON CONFLICT (id) DO UPDATE SET title = CASE WHEN EXCLUDED.updated_at > updated_at THEN EXCLUDED.title ELSE title END`, - }, - { - name: "lease upsert with (issue_id) target (bd-lrgn1)", - in: `INSERT INTO leases (issue_id, holder, granted_at, lease_expires_at, heartbeat_at) VALUES (?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE holder = VALUES(holder), lease_expires_at = VALUES(lease_expires_at)`, - want: `INSERT INTO leases (issue_id, holder, granted_at, lease_expires_at, heartbeat_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (issue_id) DO UPDATE SET holder = EXCLUDED.holder, lease_expires_at = EXCLUDED.lease_expires_at`, - }, - { - name: "lease import-restore never-clobber-live guard (bd-lrgn1)", - in: `INSERT INTO leases (issue_id, holder) VALUES (?, ?) ON DUPLICATE KEY UPDATE holder = IF(leases.lease_expires_at >= ?, leases.holder, VALUES(holder))`, - want: `INSERT INTO leases (issue_id, holder) VALUES ($1, $2) ON CONFLICT (issue_id) DO UPDATE SET holder = CASE WHEN leases.lease_expires_at >= $3 THEN leases.holder ELSE EXCLUDED.holder END`, - }, - { - name: "child counter GREATEST upsert with (parent_id) target", - in: `INSERT INTO child_counters (parent_id, last_child) VALUES (?, ?) ON DUPLICATE KEY UPDATE last_child = GREATEST(last_child, ?)`, - want: `INSERT INTO child_counters (parent_id, last_child) VALUES ($1, $2) ON CONFLICT (parent_id) DO UPDATE SET last_child = GREATEST(last_child, $3)`, - }, - { - name: "UPDATE SET unqualifies target, keeps RHS alias reference", - in: `UPDATE issues i SET i.is_blocked = 1, i.updated_at = i.updated_at WHERE i.id = ?`, - want: `UPDATE issues i SET is_blocked = 1, updated_at = i.updated_at WHERE i.id = $1`, - }, - { - name: "REPLACE INTO -> INSERT ON CONFLICT DO UPDATE", - in: "REPLACE INTO config (`key`, value) VALUES (?, ?)", - want: `INSERT INTO config ("key", value) VALUES ($1, $2) ON CONFLICT ("key") DO UPDATE SET value = EXCLUDED.value`, - }, - { - name: "CONCAT -> || (with LIKE and escaped percent)", - in: `SELECT id FROM issues WHERE id LIKE CONCAT(?, '.%')`, - want: `SELECT id FROM issues WHERE id LIKE ($1 || '.%')`, - }, - { - name: "LOCATE(CONCAT(...)) nested -> POSITION(... IN ...)", - in: `SELECT 1 WHERE LOCATE(CONCAT(',', e.issue_id, ','), d.path) = 0`, - want: `SELECT 1 WHERE POSITION((',' || e.issue_id || ',') IN d.path) = 0`, - }, - { - name: "UTC_TIMESTAMP()", - in: `SELECT 1 WHERE defer_until IS NULL OR defer_until <= UTC_TIMESTAMP()`, - want: `SELECT 1 WHERE defer_until IS NULL OR defer_until <= (CURRENT_TIMESTAMP AT TIME ZONE 'utc')`, - }, - { - name: "literal JSON gate path", - in: `SELECT 1 WHERE JSON_UNQUOTE(JSON_EXTRACT(d.metadata, '$.gate')) = 'any-children'`, - want: `SELECT 1 WHERE (d.metadata #>> '{gate}') = 'any-children'`, - }, - { - name: "binary label collation", - in: `SELECT label FROM labels WHERE issue_id = ? ORDER BY label`, - want: `SELECT label FROM labels WHERE issue_id = $1 ORDER BY label COLLATE "C"`, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got, err := Translate(tc.in) - if err != nil { - t.Fatalf("Translate error: %v", err) - } - if got != tc.want { - t.Errorf("Translate mismatch\n in: %s\n got: %s\n want: %s", tc.in, got, tc.want) - } - }) - } -} diff --git a/internal/storage/postgres/collation.go b/internal/storage/postgres/collation.go deleted file mode 100644 index d3f8ee205..000000000 --- a/internal/storage/postgres/collation.go +++ /dev/null @@ -1,49 +0,0 @@ -package postgres - -import ( - "context" - "database/sql" - "fmt" - "os" - "strings" -) - -// collationParityOK reports whether a Postgres database-level collation orders text -// by byte / Unicode code point — the ordering the shared SQL layer needs to match -// Dolt's utf8mb4_0900_bin oracle. C and POSIX are exact code-point order, and the -// C.* family (e.g. C.UTF-8) also sorts by code point. glibc/ICU locales such as -// en_US.UTF-8 sort linguistically and silently diverge from Dolt. -func collationParityOK(collation string) bool { - c := strings.ToUpper(strings.TrimSpace(collation)) - return c == "C" || c == "POSIX" || strings.HasPrefix(c, "C.") -} - -// databaseCollation returns the connected database's lc_collate. -func databaseCollation(ctx context.Context, db *sql.DB) (string, error) { - var collation string - if err := db.QueryRowContext(ctx, "SHOW lc_collate").Scan(&collation); err != nil { - return "", err - } - return collation, nil -} - -// warnOnCollationDivergence prints a prominent stderr warning when the connected -// database is not code-point ordered. Text columns use the database default -// collation — per-column COLLATE "C" was dropped because it breaks the shared -// recursive CTEs (see schema.go) — so a non-C/POSIX database makes every text -// ORDER BY the shared layer emits (bd list, bd ready, label ordering) sort -// linguistically and diverge from Dolt with no error and no other signal. -// -// It is best-effort: a query failure is swallowed so the check can never block -// opening the store. Running from Provision means `bd init --backend=postgres` -// surfaces the requirement at setup and every later open re-warns until the -// operator recreates the database with a code-point collation. This mirrors the -// Dolt backend's open-time stderr warnings. -func warnOnCollationDivergence(ctx context.Context, db *sql.DB) { - collation, err := databaseCollation(ctx, db) - if err != nil || collationParityOK(collation) { - return - } - fmt.Fprintf(os.Stderr, "Warning: Postgres database collation is %q, not C/POSIX. Text ORDER BY (bd list, bd ready, labels) sorts linguistically and diverges from Dolt's code-point order.\n", collation) - fmt.Fprintln(os.Stderr, " Recreate the database with a code-point collation, e.g. CREATE DATABASE ... TEMPLATE template0 LC_COLLATE 'C' LC_CTYPE 'C';") -} diff --git a/internal/storage/postgres/collation_test.go b/internal/storage/postgres/collation_test.go deleted file mode 100644 index 472100fae..000000000 --- a/internal/storage/postgres/collation_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package postgres - -import "testing" - -func TestCollationParityOK(t *testing.T) { - cases := []struct { - collation string - want bool - }{ - {"C", true}, - {"POSIX", true}, - {"c", true}, // case-insensitive - {"posix", true}, // case-insensitive - {" C ", true}, // trimmed - {"C.UTF-8", true}, // C.* family sorts by code point - {"C.utf8", true}, - {"en_US.UTF-8", false}, - {"en_US.utf8", false}, - {"en_GB.UTF-8", false}, - {"und-x-icu", false}, // ICU root - {"de_DE.UTF-8", false}, - {"", false}, - } - for _, tc := range cases { - if got := collationParityOK(tc.collation); got != tc.want { - t.Errorf("collationParityOK(%q) = %v, want %v", tc.collation, got, tc.want) - } - } -} diff --git a/internal/storage/postgres/completeness_test.go b/internal/storage/postgres/completeness_test.go deleted file mode 100644 index e95bc0182..000000000 --- a/internal/storage/postgres/completeness_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package postgres - -import ( - "os" - "regexp" - "sort" - "testing" - - "github.com/steveyegge/beads/internal/storage/conformance" -) - -// legitimatelyUnsupported is the EXPLICIT denominator of every storage.DoltStorage -// method the Postgres wedge deliberately does not implement, each with a reason. -// -// The interface-completeness gate (TestInterfaceCompleteness) asserts the -// generated capability shell (unsupported_gen.go) contains EXACTLY these methods -// — no more, no less. This is the guard the red-team had to stand in for: a -// method that silently resolves to the typed-unsupported shell (the class that -// bit us: DeleteIssues, GetCustomStatuses, the batch reads, …) fails this test -// at build time instead of failing a user at runtime. Adding a method to the -// shell without an entry here is a hard error that forces the triage question: -// "implement it in sqlkit, or record here why the wedge legitimately omits it." -var legitimatelyUnsupported = map[string]string{ - // VersionControl — the Dolt commit graph. The wedge's whole premise is - // "bd on Postgres, history/version-control simply absent". - "Branch": "VC: Dolt branch", "Checkout": "VC: Dolt checkout", - "CurrentBranch": "VC", "DeleteBranch": "VC", "ListBranches": "VC", - // Commit/CommitWithConfig/CommitMergeResolution are NO-OPS on Postgres (data - // is already durable per-tx), not unsupported — implemented on postgres.Store. - "CommitExists": "VC", "GetCurrentCommit": "VC: Dolt hash", "Status": "VC", - "Log": "VC: dolt_log", "Merge": "VC", "GetConflicts": "VC", "ResolveConflicts": "VC", - - // HistoryViewer — Dolt time-travel. - "History": "history: dolt_log", "AsOf": "history: as-of ref", "Diff": "history: dolt_diff", - - // RemoteStore — Dolt remotes. - "AddRemote": "remote", "RemoveRemote": "remote", "HasRemote": "remote", "ListRemotes": "remote", - "Push": "remote", "Pull": "remote", "ForcePush": "remote", "PushRemote": "remote", - "PullRemote": "remote", "Fetch": "remote", "PushTo": "remote", "PullFrom": "remote", - - // SyncStore — Dolt peer sync. - "Sync": "sync", "SyncStatus": "sync", - - // FederationStore — Dolt-remote-backed peer table. - "AddFederationPeer": "federation", "GetFederationPeer": "federation", - "ListFederationPeers": "federation", "RemoveFederationPeer": "federation", - - // CompactionStore — bound to Dolt commit hashes. - "CheckEligibility": "compaction", "ApplyCompaction": "compaction", - "GetTier1Candidates": "compaction", "GetTier2Candidates": "compaction", - "SnapshotIssue": "compaction", "GetCompactionSnapshot": "compaction", "RestoreFromSnapshot": "compaction", - - // MergeSlot — the Dolt merge-serialization primitive. - "MergeSlotCreate": "merge-slot", "MergeSlotCheck": "merge-slot", - "MergeSlotAcquire": "merge-slot", "MergeSlotRelease": "merge-slot", -} - -var shellMethodRe = regexp.MustCompile(`func \(unsupportedDoltStorage\) ([A-Za-z0-9]+)\(`) - -// TestInterfaceCompleteness is the interface-completeness gate: the generated -// shell must equal the audited legitimatelyUnsupported set, both directions. -func TestInterfaceCompleteness(t *testing.T) { - src, err := os.ReadFile("unsupported_gen.go") - if err != nil { - t.Fatalf("read unsupported_gen.go: %v", err) - } - shell := map[string]bool{} - for _, m := range shellMethodRe.FindAllStringSubmatch(string(src), -1) { - shell[m[1]] = true - } - if len(shell) == 0 { - t.Fatal("parsed 0 shell methods — regex or file drift") - } - - // (1) No SILENT gap: every shell method must be explicitly justified. - var unjustified []string - for m := range shell { - if _, ok := legitimatelyUnsupported[m]; !ok { - unjustified = append(unjustified, m) - } - } - sort.Strings(unjustified) - for _, m := range unjustified { - t.Errorf("method %q resolves to the typed-unsupported shell but is NOT in legitimatelyUnsupported: implement it on *sqlkit.Store, or add it here with a reason", m) - } - - // (2) No STALE allowlist: every justified method must still be in the shell - // (else it was implemented and the entry should be removed). - var stale []string - for m := range legitimatelyUnsupported { - if !shell[m] { - stale = append(stale, m) - } - } - sort.Strings(stale) - for _, m := range stale { - t.Errorf("method %q is in legitimatelyUnsupported but no longer in the shell (implemented?): remove the allowlist entry", m) - } -} - -// TestUnsupportedContract is the behavioral complement to TestInterfaceCompleteness: -// every allowlisted method must actually return a typed storage.ErrUnsupported when -// called, not panic or return a different error. DB-free — the generated stubs ignore -// their receiver, so a zero-value store answers them. -func TestUnsupportedContract(t *testing.T) { - conformance.RunUnsupportedContract(t, &Store{}, legitimatelyUnsupported) -} diff --git a/internal/storage/postgres/conformance_test.go b/internal/storage/postgres/conformance_test.go deleted file mode 100644 index d7eb19446..000000000 --- a/internal/storage/postgres/conformance_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package postgres - -import ( - "context" - "fmt" - "os" - "sync/atomic" - "testing" - "time" - - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/storage/conformance" - "github.com/steveyegge/beads/internal/storage/pgdialect" -) - -// TestConformance runs bd's backend-agnostic storage conformance suite -// (internal/storage/conformance, ~40 encoded behavior tests) against the -// Postgres wedge. This is the real conformance measurement — far broader than -// the smoke. Every failure is a PG gap: either an allowlisted-unsupported method -// (expected — see completeness_test.go) or a latent divergence nothing else -// caught (the interesting ones). Gated on BEADS_PG_TEST_URL. -func TestConformance(t *testing.T) { - url := os.Getenv("BEADS_PG_TEST_URL") - if url == "" { - t.Skip("BEADS_PG_TEST_URL not set") - } - conformance.RunAll(t, pgConformanceFactory(url)) -} - -// TestDeferredReads runs the shared deferred non-version-control reads (statistics, -// external-ref, stale) that the wedge now implements through issueops, as a GREEN -// gate. RunAll stays the fail-loud measurement (red on genuinely Dolt-only methods -// like slots), so this focused gate is what conformance.sh runs for the wedge. -// Gated on BEADS_PG_TEST_URL. -func TestDeferredReads(t *testing.T) { - url := os.Getenv("BEADS_PG_TEST_URL") - if url == "" { - t.Skip("BEADS_PG_TEST_URL not set") - } - conformance.RunDeferredReads(t, pgConformanceFactory(url)) -} - -// TestPortableMethods runs the full portable-method behavior contract (molecule/ -// repo-mtime/streams/counts/comment/rekey/batch). Red until the methods are wired into -// sqlkit; green after. Gated on BEADS_PG_TEST_URL. -func TestPortableMethods(t *testing.T) { - url := os.Getenv("BEADS_PG_TEST_URL") - if url == "" { - t.Skip("BEADS_PG_TEST_URL not set") - } - conformance.RunPortableMethods(t, pgConformanceFactory(url)) -} - -// pgConformanceFactory provisions a fresh, isolated schema per sub-test against the -// Postgres server at url, seeded with issue_prefix as `bd init` leaves it. -func pgConformanceFactory(url string) conformance.Factory { - base := time.Now().UnixNano() - var seq int64 - return func(t *testing.T) storage.DoltStorage { - ctx := context.Background() - schema := fmt.Sprintf("conf_%d_%d", base, atomic.AddInt64(&seq, 1)) - - raw, err := pgdialect.OpenRaw(url, schema) - if err != nil { - t.Fatalf("OpenRaw: %v", err) - } - if err := InitSchema(ctx, raw, schema); err != nil { - _ = raw.Close() - t.Fatalf("InitSchema: %v", err) - } - _ = raw.Close() - - st, err := New(ctx, Config{DSN: url, Schema: schema}) - if err != nil { - t.Fatalf("New: %v", err) - } - // The conformance factory contract wants an init'd store (issue_prefix - // set), as `bd init` leaves it. No Commit — PG has no commit graph. - if err := st.SetConfig(ctx, "issue_prefix", "test"); err != nil { - t.Fatalf("SetConfig(issue_prefix): %v", err) - } - t.Cleanup(func() { - _, _ = st.DB().ExecContext(context.Background(), - fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) - _ = st.Close() - }) - return st - } -} diff --git a/internal/storage/postgres/credential.go b/internal/storage/postgres/credential.go deleted file mode 100644 index f5eb12cf9..000000000 --- a/internal/storage/postgres/credential.go +++ /dev/null @@ -1,67 +0,0 @@ -package postgres - -import ( - "context" - "fmt" - "os" - - "github.com/steveyegge/beads/internal/configfile" - "github.com/steveyegge/beads/internal/creds" - "github.com/steveyegge/beads/internal/storage/pgdialect" -) - -// resolveDSNCredential returns baseDSN with a credential resolved and placed, ready to -// connect. The precedence, high to low: -// -// - BEADS_PG_CREDENTIAL_COMMAND set is a hard error: an identity credential is -// presented as a username to a gateway, and this backend connects directly; -// - the DSN already carries a password (a full BEADS_POSTGRES_URL, or an explicit -// --pg-url) — it wins outright, no ladder is applied; -// - BEADS_PG_PASSWORD_COMMAND — a credential command (e.g. Vault / RDS-IAM / GCP-IAM); -// run at open time, cached until near expiry; -// - BEADS_PG_PASSWORD — a static password; -// - the credentials file [host:port] ($BEADS_CREDENTIALS_FILE or -// ~/.config/beads/credentials); -// - nothing configured — baseDSN is returned untouched so pgx's own libpq -// fallbacks (PGPASSWORD, ~/.pgpass, PGPASSFILE) still apply. -// -// It fails closed: if a configured credential command errors, the open aborts — it -// never silently downgrades to a lower rung. -func resolveDSNCredential(ctx context.Context, cfg *configfile.Config, baseDSN string) (string, error) { - if os.Getenv("BEADS_PG_CREDENTIAL_COMMAND") != "" { - return "", fmt.Errorf("postgres: BEADS_PG_CREDENTIAL_COMMAND is reserved: an identity credential is presented as a username to a gateway, and the postgres backend connects directly to the database with no gateway to present it to; use BEADS_PG_PASSWORD_COMMAND for a helper that produces a password") - } - // host:port keys the credentials-file lookup; ok=false (unix socket / unparseable) - // leaves the file rung not-configured (zero Host/Port), never failing. - host, port, _ := pgdialect.HostPort(baseDSN) - return resolveDSNWithSources(ctx, baseDSN, - creds.CommandSource{Command: cfg.GetPostgresPasswordCommand(), Kind: creds.KindSecret, Label: "BEADS_PG_PASSWORD_COMMAND"}, - creds.EnvSource{Var: "BEADS_PG_PASSWORD"}, - creds.FileSource{Host: host, Port: port, Lookup: configfile.LookupCredentialsPassword, Label: "credentials-file"}, - ) -} - -// resolveDSNWithSources is the testable core of resolveDSNCredential: it applies the -// given credential ladder to baseDSN. A DSN that already carries a password wins -// outright; otherwise the first configured source supplies the credential, and -// nothing-configured leaves the DSN untouched for pgx's libpq fallbacks. -func resolveDSNWithSources(ctx context.Context, baseDSN string, sources ...creds.Source) (string, error) { - if pgdialect.HasPassword(baseDSN) { - return baseDSN, nil - } - cred, ok, err := creds.ResolveLadder(ctx, sources...) - if err != nil { - return "", err - } - if !ok { - return baseDSN, nil - } - // The SQL backends take a secret in the password slot; an identity credential - // (presented as a username to a gateway) has no home on a direct connection, so - // refuse it loudly rather than land a token where a password belongs. Defense in - // depth behind the BEADS_PG_CREDENTIAL_COMMAND reject above. - if cred.Kind != creds.KindSecret { - return "", fmt.Errorf("postgres: credential from %s is an identity, not a password; the postgres backend connects directly and has no gateway to present it to", cred.Source) - } - return pgdialect.WithCredential(baseDSN, cred.Username, cred.Value) -} diff --git a/internal/storage/postgres/credential_test.go b/internal/storage/postgres/credential_test.go deleted file mode 100644 index b2c87572b..000000000 --- a/internal/storage/postgres/credential_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package postgres - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/jackc/pgx/v5" - "github.com/steveyegge/beads/internal/configfile" - "github.com/steveyegge/beads/internal/creds" -) - -const baseDSN = "postgres://bts@127.0.0.1:5432/db" - -// TestMain neutralizes every ambient credential source the rungs could read, so tests -// are hermetic: the beads credentials file, plus pgx's own PGPASSWORD/~/.pgpass (which -// pgx.ParseConfig folds into a connection at parse/connect time). Tests that exercise -// the file rung override BEADS_CREDENTIALS_FILE per-test with t.Setenv. -func TestMain(m *testing.M) { - dir, err := os.MkdirTemp("", "beads-pg-cred-test") - if err != nil { - panic(err) - } - os.Setenv("BEADS_CREDENTIALS_FILE", filepath.Join(dir, "none")) - os.Unsetenv("PGPASSWORD") - os.Setenv("PGPASSFILE", filepath.Join(dir, "no-pgpass")) - code := m.Run() - _ = os.RemoveAll(dir) - os.Exit(code) -} - -// password parses out and returns the password pgx sees, failing the test on a parse error. -func password(t *testing.T, dsn string) string { - t.Helper() - cfg, err := pgx.ParseConfig(dsn) - if err != nil { - t.Fatalf("pgx.ParseConfig(%q): %v", dsn, err) - } - return cfg.Password -} - -// user parses out and returns the username pgx sees. -func user(t *testing.T, dsn string) string { - t.Helper() - cfg, err := pgx.ParseConfig(dsn) - if err != nil { - t.Fatalf("pgx.ParseConfig(%q): %v", dsn, err) - } - return cfg.User -} - -// writeCredsFile writes an INI credentials file and returns its path. -func writeCredsFile(t *testing.T, body string) string { - t.Helper() - p := filepath.Join(t.TempDir(), "credentials") - if err := os.WriteFile(p, []byte(body), 0o600); err != nil { - t.Fatal(err) - } - return p -} - -func TestResolveDSNCredentialCommand(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD_COMMAND", "printf cmd-pw") - t.Setenv("BEADS_PG_PASSWORD", "") - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "cmd-pw" { - t.Fatalf("password = %q, want cmd-pw", pw) - } -} - -func TestResolveDSNCredentialEnv(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD_COMMAND", "") - t.Setenv("BEADS_PG_PASSWORD", "env-pw") - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "env-pw" { - t.Fatalf("password = %q, want env-pw", pw) - } -} - -// The command out-ranks the static env password. -func TestResolveDSNCredentialCommandBeatsEnv(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD_COMMAND", "printf cmd-wins") - t.Setenv("BEADS_PG_PASSWORD", "env-loses") - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "cmd-wins" { - t.Fatalf("password = %q, want cmd-wins", pw) - } -} - -// A failing command aborts the open and does NOT fall through to the env password. -func TestResolveDSNCredentialFailsClosed(t *testing.T) { - // A bare token with whitespace is rejected by parseCredential — a configured error. - t.Setenv("BEADS_PG_PASSWORD_COMMAND", `printf 'access denied'`) - t.Setenv("BEADS_PG_PASSWORD", "env-would-be-wrong") - _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err == nil { - t.Fatal("expected an error (fail-closed); a broken command must not fall through to BEADS_PG_PASSWORD") - } -} - -// Nothing configured: the base DSN comes back untouched (pgx's own PGPASSWORD/.pgpass -// fallbacks then apply at connect). -func TestResolveDSNCredentialNothingConfigured(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD_COMMAND", "") - t.Setenv("BEADS_PG_PASSWORD", "") - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if got != baseDSN { - t.Fatalf("dsn = %q, want it unchanged (%q)", got, baseDSN) - } -} - -// A DSN that already carries a password wins outright — the ladder is not applied. -func TestResolveDSNCredentialExistingPasswordWins(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD_COMMAND", "printf should-not-run") - t.Setenv("BEADS_PG_PASSWORD", "should-not-run") - withPw := "postgres://bts:already-here@127.0.0.1:5432/db" - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, withPw) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "already-here" { - t.Fatalf("password = %q, want already-here (existing password must win)", pw) - } -} - -// The credentials file resolves the password when no command or env is set. -func TestResolveDSNCredentialFile(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD_COMMAND", "") - t.Setenv("BEADS_PG_PASSWORD", "") - t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:5432]\npassword=file-pw\n")) - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "file-pw" { - t.Fatalf("password = %q, want file-pw", pw) - } -} - -// The env password out-ranks the credentials file. -func TestResolveDSNCredentialEnvBeatsFile(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD_COMMAND", "") - t.Setenv("BEADS_PG_PASSWORD", "env-pw") - t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:5432]\npassword=file-pw\n")) - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if pw := password(t, got); pw != "env-pw" { - t.Fatalf("password = %q, want env-pw (env out-ranks file)", pw) - } -} - -// A file section for a different endpoint does not match: the DSN is untouched. -func TestResolveDSNCredentialFileWrongSectionMisses(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD_COMMAND", "") - t.Setenv("BEADS_PG_PASSWORD", "") - t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:9999]\npassword=file-pw\n")) - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if got != baseDSN { - t.Fatalf("dsn = %q, want it unchanged (wrong section must not match)", got) - } -} - -// A broken command fails closed even when the file could supply a valid password. -func TestResolveDSNCredentialBrokenCommandNotRescuedByFile(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD_COMMAND", "false") - t.Setenv("BEADS_PG_PASSWORD", "") - t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:5432]\npassword=file-pw\n")) - if _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN); err == nil { - t.Fatal("expected fail-closed; the credentials file must not rescue a broken command") - } -} - -// A set BEADS_PG_CREDENTIAL_COMMAND is a hard error, pre-empting even a -// password-bearing DSN and any lower rung. -func TestResolveDSNCredentialRejectsReservedIdentityVar(t *testing.T) { - t.Setenv("BEADS_PG_CREDENTIAL_COMMAND", "get-credential") - t.Setenv("BEADS_PG_PASSWORD", "would-otherwise-work") - _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, "postgres://bts:has-pw@127.0.0.1:5432/db") - if err == nil || !strings.Contains(err.Error(), "BEADS_PG_CREDENTIAL_COMMAND") { - t.Fatalf("expected a reserved-var rejection naming BEADS_PG_CREDENTIAL_COMMAND, got %v", err) - } -} - -// staticSource is a test Source with a fixed credential, used to exercise placement -// and the identity-refusal guard directly. -type staticSource struct{ cred creds.Credential } - -func (s staticSource) Name() string { return "test" } -func (s staticSource) Resolve(context.Context) (creds.Credential, bool, error) { - return s.cred, true, nil -} - -// An identity credential (KindIdentity) has no home on a direct SQL connection and -// must be refused, never landed in the password slot. -func TestResolveDSNCredentialRefusesIdentity(t *testing.T) { - src := staticSource{cred: creds.Credential{Value: "an-identity-token", Kind: creds.KindIdentity, Source: "test"}} - _, err := resolveDSNWithSources(context.Background(), baseDSN, src) - if err == nil || !strings.Contains(err.Error(), "identity") { - t.Fatalf("expected an identity-refusal error, got %v", err) - } -} - -// A credential carrying a username (a Vault dynamic user/password pair) places both. -func TestResolveDSNCredentialPlacesUsername(t *testing.T) { - src := staticSource{cred: creds.Credential{Value: "pw", Username: "dyn", Kind: creds.KindSecret}} - got, err := resolveDSNWithSources(context.Background(), baseDSN, src) - if err != nil { - t.Fatal(err) - } - if u := user(t, got); u != "dyn" { - t.Fatalf("user = %q, want dyn", u) - } - if pw := password(t, got); pw != "pw" { - t.Fatalf("password = %q, want pw", pw) - } -} - -// An empty username leaves the DSN's user untouched. -func TestResolveDSNCredentialEmptyUsernameKeepsDSNUser(t *testing.T) { - src := staticSource{cred: creds.Credential{Value: "pw", Username: "", Kind: creds.KindSecret}} - got, err := resolveDSNWithSources(context.Background(), baseDSN, src) - if err != nil { - t.Fatal(err) - } - if u := user(t, got); u != "bts" { - t.Fatalf("user = %q, want bts (DSN user preserved)", u) - } -} - -// End-to-end: a JSON envelope carrying a username places both user and password. -func TestResolveDSNCredentialEnvelopeUsernameEndToEnd(t *testing.T) { - t.Setenv("BEADS_PG_PASSWORD_COMMAND", `printf '{"token":"pw","username":"dyn"}'`) - t.Setenv("BEADS_PG_PASSWORD", "") - got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) - if err != nil { - t.Fatal(err) - } - if u, pw := user(t, got), password(t, got); u != "dyn" || pw != "pw" { - t.Fatalf("(user,password) = (%q,%q), want (dyn,pw)", u, pw) - } -} diff --git a/internal/storage/postgres/dialect.go b/internal/storage/postgres/dialect.go deleted file mode 100644 index 1a5707f13..000000000 --- a/internal/storage/postgres/dialect.go +++ /dev/null @@ -1,22 +0,0 @@ -package postgres - -import ( - "context" - "database/sql" - - "github.com/steveyegge/beads/internal/storage/pgdialect" -) - -// pgDialect is the Postgres SQL flavor: it opens a *sql.DB whose driver -// translates bd's canonical (MySQL-dialect) SQL to Postgres, and pins the -// workspace schema via search_path. -type pgDialect struct { - dsn string - searchPath string -} - -func (d pgDialect) Name() string { return "postgres" } - -func (d pgDialect) Open(_ context.Context) (*sql.DB, error) { - return pgdialect.Open(d.dsn, d.searchPath) -} diff --git a/internal/storage/postgres/fromconfig.go b/internal/storage/postgres/fromconfig.go deleted file mode 100644 index d43f224cf..000000000 --- a/internal/storage/postgres/fromconfig.go +++ /dev/null @@ -1,68 +0,0 @@ -package postgres - -import ( - "context" - "fmt" - - "github.com/steveyegge/beads/internal/configfile" - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/storage/pgdialect" -) - -// NewFromConfig opens the Postgres backend for a workspace. It reads the base DSN -// and per-workspace schema from .beads/metadata.json, resolves the password through -// the credential ladder (command > env > credentials file, fail-closed) and places -// it into the DSN — never persisting it — then applies the schema over a raw -// non-translating connection and returns the store. This is the factory arm cmd/bd -// dispatches to when metadata.json has backend="postgres". -func NewFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { - cfg, err := configfile.Load(beadsDir) - if err != nil { - return nil, fmt.Errorf("postgres: load config: %w", err) - } - dsn := cfg.GetPostgresDSN() - if dsn == "" { - return nil, fmt.Errorf("postgres: no DSN (set postgres_dsn in metadata.json, or BEADS_POSTGRES_URL)") - } - dsn, err = resolveDSNCredential(ctx, cfg, dsn) - if err != nil { - return nil, fmt.Errorf("postgres: resolve credential: %w", err) - } - schema := cfg.GetPostgresSchema() - if schema == "" { - return nil, fmt.Errorf("postgres: no schema (set postgres_schema in metadata.json)") - } - return Provision(ctx, dsn, schema) -} - -// Provision opens the Postgres backend from an explicit DSN and schema: it applies -// the schema (InitSchema — create-schema, DDL, config seeds, version stamp; all -// idempotent) over a raw non-translating connection, then returns the store over the -// translating driver. dsn may carry a password; it is used only to connect and is -// never persisted. `bd init` calls this directly (before metadata.json exists, using -// the flag-provided DSN); NewFromConfig calls it after resolving DSN/schema from -// metadata. DDL runs raw because the translating driver would mangle the $$-quoted -// function bodies and treat DDL as workload SQL. -func Provision(ctx context.Context, dsn, schema string) (storage.DoltStorage, error) { - if dsn == "" { - return nil, fmt.Errorf("postgres: empty DSN") - } - if schema == "" { - return nil, fmt.Errorf("postgres: empty schema") - } - raw, err := pgdialect.OpenRaw(dsn, schema) - if err != nil { - return nil, fmt.Errorf("postgres: open (raw): %w", err) - } - if err := InitSchema(ctx, raw, schema); err != nil { - _ = raw.Close() - return nil, fmt.Errorf("postgres: init schema: %w", err) - } - // Byte-order parity with Dolt depends on the database collation (text columns - // use the DB default; per-column COLLATE "C" breaks the shared recursive CTEs). - // Warn loudly when it is not code-point ordered, at init and on every open. - warnOnCollationDivergence(ctx, raw) - _ = raw.Close() - - return New(ctx, Config{DSN: dsn, Schema: schema}) -} diff --git a/internal/storage/postgres/repro_instr_test.go b/internal/storage/postgres/repro_instr_test.go deleted file mode 100644 index 6b1478c2e..000000000 --- a/internal/storage/postgres/repro_instr_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package postgres - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/steveyegge/beads/internal/types" -) - -// Temporary red-team repro: create with NO explicit ID (default hash-ID path) -// which exercises GetAdaptiveIDLengthTx's INSTR() query. -func TestPGReproHashIDCreate(t *testing.T) { - url := os.Getenv("BEADS_PG_TEST_URL") - if url == "" { - t.Skip("BEADS_PG_TEST_URL not set") - } - schema := fmt.Sprintf("repro_%d", time.Now().UnixNano()) - ctx := context.Background() - - st, err := New(ctx, Config{DSN: url, Schema: schema}) - if err != nil { - t.Fatalf("New: %v", err) - } - defer st.Close() - if err := InitSchema(ctx, st.DB(), schema); err != nil { - t.Fatalf("InitSchema: %v", err) - } - t.Cleanup(func() { - _, _ = st.DB().ExecContext(context.Background(), fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) - }) - if err := st.SetConfig(ctx, "issue_prefix", "repro"); err != nil { - t.Fatalf("SetConfig: %v", err) - } - - iss := &types.Issue{ - Title: "no explicit id", IssueType: types.IssueType("task"), - Status: types.Status("open"), Priority: 2, CreatedBy: "tester", - } - if err := st.CreateIssue(ctx, iss, "tester"); err != nil { - t.Fatalf("CreateIssue(hash-id): %v", err) - } - t.Logf("created id=%s", iss.ID) -} diff --git a/internal/storage/postgres/schema.go b/internal/storage/postgres/schema.go deleted file mode 100644 index 152cfa09a..000000000 --- a/internal/storage/postgres/schema.go +++ /dev/null @@ -1,685 +0,0 @@ -package postgres - -import ( - "context" - "database/sql" - "errors" - "fmt" - "hash/fnv" - "regexp" - "strings" -) - -// schemaAdvisoryLockKey derives a stable Postgres advisory-lock key from a -// workspace schema name. Distinct schemas map to distinct keys so per-workspace -// InitSchema calls serialize against themselves but never against each other. -func schemaAdvisoryLockKey(schema string) int64 { - h := fnv.New64a() - _, _ = h.Write([]byte("bd.initschema:" + schema)) - return int64(h.Sum64()) -} - -// ddl is the Postgres backend schema, embedded verbatim from -// internal/storage/postgres/schema.sql. It is applied statement-by-statement by -// InitSchema because the pgx extended protocol forbids multiple statements per -// Exec. -const ddl = `-- bd Postgres backend schema. --- Machine-translated from Dolt ` + "`" + `SHOW CREATE TABLE` + "`" + ` output at HEAD (all --- migrations applied), using the conformance type map documented in --- docs/architecture/storage-backends.md: --- varchar/char/text/longtext -> text (see collation note below) --- datetime -> timestamp(0) (naive whole-second UTC) --- tinyint(1) -> smallint --- json DEFAULT (json_object()) -> jsonb DEFAULT '{}' --- int -> integer, bigint -> bigint --- ON UPDATE CURRENT_TIMESTAMP -> (no trigger; the shared layer always assigns --- updated_at explicitly on every UPDATE) --- inline KEY / UNIQUE KEY -> CREATE [UNIQUE] INDEX after each table --- All statements are idempotent (IF NOT EXISTS / OR REPLACE); run with --- search_path pinned to the per-workspace schema. --- Views ready_issues / blocked_issues intentionally omitted (unused). --- --- Collation note: Dolt's text columns are utf8mb4_0900_bin (byte / code-point --- order). Per-column COLLATE "C" was dropped from this DDL because it breaks the --- shared layer's parameter-seeded recursive CTEs (ERROR: recursive query column --- has collation "default" in non-recursive term but collation "C" overall). Text --- columns therefore use the database's default collation, so byte-order parity --- with Dolt is a documented wedge gap that must be guaranteed at DEPLOYMENT time --- by creating the database with LC_COLLATE 'C' (CREATE DATABASE ... TEMPLATE --- template0 LC_COLLATE 'C' LC_CTYPE 'C'), NOT per-column. Under any glibc/ICU --- en_US.* collation, every text ORDER BY the shared layer emits diverges from --- Dolt's code-point order. - --- ============================================================ issues - -CREATE TABLE IF NOT EXISTS issues ( - id text NOT NULL, - content_hash text, - title text NOT NULL, - description text NOT NULL, - design text NOT NULL, - acceptance_criteria text NOT NULL, - notes text NOT NULL, - status text NOT NULL DEFAULT 'open', - priority integer NOT NULL DEFAULT 2, - issue_type text NOT NULL DEFAULT 'task', - assignee text, - estimated_minutes integer, - created_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - created_by text DEFAULT '', - owner text DEFAULT '', - updated_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - closed_at timestamp(0), - closed_by_session text DEFAULT '', - external_ref text, - spec_id text, - compaction_level integer DEFAULT 0, - compacted_at timestamp(0), - compacted_at_commit text, - original_size integer, - sender text DEFAULT '', - ephemeral smallint DEFAULT 0, - wisp_type text DEFAULT '', - pinned smallint DEFAULT 0, - is_template smallint DEFAULT 0, - mol_type text DEFAULT '', - work_type text DEFAULT 'mutex', - source_system text DEFAULT '', - metadata jsonb DEFAULT '{}', - source_repo text DEFAULT '', - close_reason text DEFAULT '', - event_kind text DEFAULT '', - actor text DEFAULT '', - target text DEFAULT '', - payload text DEFAULT '', - await_type text DEFAULT '', - await_id text DEFAULT '', - timeout_ns bigint DEFAULT 0, - waiters text DEFAULT '', - hook_bead text DEFAULT '', - role_bead text DEFAULT '', - agent_state text DEFAULT '', - last_activity timestamp(0), - role_type text DEFAULT '', - rig text DEFAULT '', - due_at timestamp(0), - defer_until timestamp(0), - no_history smallint DEFAULT 0, - started_at timestamp(0), - is_blocked smallint NOT NULL DEFAULT 0, - row_lock bigint NOT NULL DEFAULT 0, - PRIMARY KEY (id) -); - -CREATE INDEX IF NOT EXISTS idx_issues_assignee ON issues (assignee); -CREATE INDEX IF NOT EXISTS idx_issues_created_at ON issues (created_at); -CREATE INDEX IF NOT EXISTS idx_issues_defer_until ON issues (defer_until); -CREATE INDEX IF NOT EXISTS idx_issues_external_ref ON issues (external_ref); -CREATE INDEX IF NOT EXISTS idx_issues_is_blocked ON issues (is_blocked, status); -CREATE INDEX IF NOT EXISTS idx_issues_issue_type ON issues (issue_type); -CREATE INDEX IF NOT EXISTS idx_issues_priority ON issues (priority); -CREATE INDEX IF NOT EXISTS idx_issues_spec_id ON issues (spec_id); -CREATE INDEX IF NOT EXISTS idx_issues_status_updated_at ON issues (status, updated_at); - --- ============================================================ leases --- Ephemeral claim leases (bd-lrgn1): node-local, never exported by backup. --- One row per live claim; see issueops.UpsertLeaseInTx for the invariant. - -CREATE TABLE IF NOT EXISTS leases ( - issue_id text NOT NULL, - holder text NOT NULL, - granted_at timestamp(0) NOT NULL, - lease_expires_at timestamp(0) NOT NULL, - heartbeat_at timestamp(0) NOT NULL, - PRIMARY KEY (issue_id) -); - -CREATE INDEX IF NOT EXISTS idx_leases_expires ON leases (lease_expires_at); - --- ============================================================ wisps - -CREATE TABLE IF NOT EXISTS wisps ( - id text NOT NULL, - content_hash text, - title text NOT NULL, - description text NOT NULL DEFAULT '', - design text NOT NULL DEFAULT '', - acceptance_criteria text NOT NULL DEFAULT '', - notes text NOT NULL DEFAULT '', - status text NOT NULL DEFAULT 'open', - priority integer NOT NULL DEFAULT 2, - issue_type text NOT NULL DEFAULT 'task', - assignee text, - estimated_minutes integer, - created_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - created_by text DEFAULT '', - owner text DEFAULT '', - updated_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - closed_at timestamp(0), - closed_by_session text DEFAULT '', - external_ref text, - spec_id text, - compaction_level integer DEFAULT 0, - compacted_at timestamp(0), - compacted_at_commit text, - original_size integer, - sender text DEFAULT '', - ephemeral smallint DEFAULT 0, - wisp_type text DEFAULT '', - pinned smallint DEFAULT 0, - is_template smallint DEFAULT 0, - mol_type text DEFAULT '', - work_type text DEFAULT 'mutex', - source_system text DEFAULT '', - metadata jsonb DEFAULT '{}', - source_repo text DEFAULT '', - close_reason text DEFAULT '', - event_kind text DEFAULT '', - actor text DEFAULT '', - target text DEFAULT '', - payload text DEFAULT '', - await_type text DEFAULT '', - await_id text DEFAULT '', - timeout_ns bigint DEFAULT 0, - waiters text DEFAULT '', - hook_bead text DEFAULT '', - role_bead text DEFAULT '', - agent_state text DEFAULT '', - last_activity timestamp(0), - role_type text DEFAULT '', - rig text DEFAULT '', - due_at timestamp(0), - defer_until timestamp(0), - no_history smallint DEFAULT 0, - started_at timestamp(0), - is_blocked smallint NOT NULL DEFAULT 0, - row_lock bigint NOT NULL DEFAULT 0, - PRIMARY KEY (id) -); - -CREATE INDEX IF NOT EXISTS idx_wisps_assignee ON wisps (assignee); -CREATE INDEX IF NOT EXISTS idx_wisps_created_at ON wisps (created_at); -CREATE INDEX IF NOT EXISTS idx_wisps_external_ref ON wisps (external_ref); -CREATE INDEX IF NOT EXISTS idx_wisps_is_blocked ON wisps (is_blocked, status); -CREATE INDEX IF NOT EXISTS idx_wisps_issue_type ON wisps (issue_type); -CREATE INDEX IF NOT EXISTS idx_wisps_priority ON wisps (priority); -CREATE INDEX IF NOT EXISTS idx_wisps_spec_id ON wisps (spec_id); -CREATE INDEX IF NOT EXISTS idx_wisps_status ON wisps (status); - --- ============================================================ helper functions --- No updated_at trigger: MySQL's ON UPDATE CURRENT_TIMESTAMP is intentionally --- NOT reproduced. The shared layer assigns updated_at explicitly on every issues --- / wisps UPDATE, including the is_blocked mark/unmark idiom that self-assigns --- (SET updated_at = updated_at) precisely to SUPPRESS the auto-bump. A BEFORE --- UPDATE trigger cannot see the statement's SET list, so it would stomp those --- deliberately-suppressed writes to now() and corrupt synced timestamps on the --- hottest dependency-state path; omitting it keeps PG identical to Dolt. - --- bd_mysql_jsonkey normalizes a MySQL JSON path to the bare top-level key the --- translator binds for dynamic metadata filters: '$.sprint' -> sprint, --- '$."gc.routed_to"' -> gc.routed_to. The translator rewrites --- JSON_EXTRACT(metadata, ?) -> metadata -> bd_mysql_jsonkey(?) and --- JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) -> metadata ->> bd_mysql_jsonkey(?). --- storage.JSONMetadataPath always quotes dotted keys into one top-level segment, --- so a single-key ->/->> lookup is exactly conformant with Dolt. -CREATE OR REPLACE FUNCTION bd_mysql_jsonkey(p text) RETURNS text LANGUAGE sql IMMUTABLE AS $$ SELECT CASE WHEN p LIKE '$."%"' THEN substring(p from 4 for length(p)-4) ELSE substring(p from 3) END $$; - --- ============================================================ labels - -CREATE TABLE IF NOT EXISTS labels ( - issue_id text NOT NULL, - label text NOT NULL, - PRIMARY KEY (issue_id, label), - CONSTRAINT fk_labels_issue FOREIGN KEY (issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE -); - -CREATE INDEX IF NOT EXISTS idx_labels_label ON labels (label); - --- ============================================================ wisp_labels - -CREATE TABLE IF NOT EXISTS wisp_labels ( - issue_id text NOT NULL, - label text NOT NULL, - PRIMARY KEY (issue_id, label), - CONSTRAINT fk_wisp_labels_issue FOREIGN KEY (issue_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE -); - -CREATE INDEX IF NOT EXISTS idx_wisp_labels_label ON wisp_labels (label); - --- ============================================================ dependencies - -CREATE TABLE IF NOT EXISTS dependencies ( - id text NOT NULL, - issue_id text NOT NULL, - type text NOT NULL DEFAULT 'blocks', - created_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - created_by text NOT NULL, - metadata jsonb DEFAULT '{}', - thread_id text DEFAULT '', - depends_on_issue_id text, - depends_on_wisp_id text, - depends_on_external text, - PRIMARY KEY (id), - CONSTRAINT fk_dep_issue FOREIGN KEY (issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT fk_dep_issue_target FOREIGN KEY (depends_on_issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT ck_dep_one_target CHECK ( - ((depends_on_issue_id IS NOT NULL)::integer - + (depends_on_wisp_id IS NOT NULL)::integer - + (depends_on_external IS NOT NULL)::integer) = 1 - ) -); - -CREATE INDEX IF NOT EXISTS idx_dep_external_target ON dependencies (depends_on_external); -CREATE INDEX IF NOT EXISTS idx_dep_issue_target ON dependencies (depends_on_issue_id); -CREATE INDEX IF NOT EXISTS idx_dep_type_external ON dependencies (type, depends_on_external); -CREATE INDEX IF NOT EXISTS idx_dep_type_issue ON dependencies (type, depends_on_issue_id); -CREATE INDEX IF NOT EXISTS idx_dep_type_wisp ON dependencies (type, depends_on_wisp_id); -CREATE INDEX IF NOT EXISTS idx_dep_wisp_target ON dependencies (depends_on_wisp_id); -CREATE INDEX IF NOT EXISTS idx_dependencies_issue ON dependencies (issue_id); -CREATE INDEX IF NOT EXISTS idx_dependencies_thread ON dependencies (thread_id); --- MySQL unique keys treat NULLs as distinct; PG's default (NULLS DISTINCT) matches. -CREATE UNIQUE INDEX IF NOT EXISTS uk_dep_external_target ON dependencies (issue_id, depends_on_external); -CREATE UNIQUE INDEX IF NOT EXISTS uk_dep_issue_target ON dependencies (issue_id, depends_on_issue_id); -CREATE UNIQUE INDEX IF NOT EXISTS uk_dep_wisp_target ON dependencies (issue_id, depends_on_wisp_id); - --- ============================================================ wisp_dependencies - -CREATE TABLE IF NOT EXISTS wisp_dependencies ( - id text NOT NULL, - issue_id text NOT NULL, - depends_on_issue_id text, - depends_on_wisp_id text, - depends_on_external text, - type text NOT NULL DEFAULT 'blocks', - created_at timestamp(0) DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - created_by text DEFAULT '', - metadata jsonb DEFAULT '{}', - thread_id text DEFAULT '', - PRIMARY KEY (id), - CONSTRAINT fk_wisp_dep_issue FOREIGN KEY (issue_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT fk_wisp_dep_issue_target FOREIGN KEY (depends_on_issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT fk_wisp_dep_wisp_target FOREIGN KEY (depends_on_wisp_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT ck_wisp_dep_one_target CHECK ( - ((depends_on_issue_id IS NOT NULL)::integer - + (depends_on_wisp_id IS NOT NULL)::integer - + (depends_on_external IS NOT NULL)::integer) = 1 - ) -); - -CREATE INDEX IF NOT EXISTS fk_wisp_dep_issue_target ON wisp_dependencies (depends_on_issue_id); -CREATE INDEX IF NOT EXISTS fk_wisp_dep_wisp_target ON wisp_dependencies (depends_on_wisp_id); -CREATE INDEX IF NOT EXISTS idx_wisp_dep_type ON wisp_dependencies (type); -CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_external ON wisp_dependencies (type, depends_on_external); -CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_issue ON wisp_dependencies (type, depends_on_issue_id); -CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_wisp ON wisp_dependencies (type, depends_on_wisp_id); -CREATE UNIQUE INDEX IF NOT EXISTS uk_wisp_dep_external_target ON wisp_dependencies (issue_id, depends_on_external); -CREATE UNIQUE INDEX IF NOT EXISTS uk_wisp_dep_issue_target ON wisp_dependencies (issue_id, depends_on_issue_id); -CREATE UNIQUE INDEX IF NOT EXISTS uk_wisp_dep_wisp_target ON wisp_dependencies (issue_id, depends_on_wisp_id); - --- ============================================================ comments - -CREATE TABLE IF NOT EXISTS comments ( - id text NOT NULL, - issue_id text NOT NULL, - author text NOT NULL, - text text NOT NULL, - created_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - PRIMARY KEY (id), - CONSTRAINT fk_comments_issue FOREIGN KEY (issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE -); - -CREATE INDEX IF NOT EXISTS idx_comments_created_at ON comments (created_at); -CREATE INDEX IF NOT EXISTS idx_comments_issue ON comments (issue_id); - --- ============================================================ wisp_comments - -CREATE TABLE IF NOT EXISTS wisp_comments ( - id text NOT NULL, - issue_id text NOT NULL, - author text DEFAULT '', - text text NOT NULL, - created_at timestamp(0) DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - PRIMARY KEY (id), - CONSTRAINT fk_wisp_comments_issue FOREIGN KEY (issue_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE -); - -CREATE INDEX IF NOT EXISTS idx_wisp_comments_issue ON wisp_comments (issue_id); - --- ============================================================ events - -CREATE TABLE IF NOT EXISTS events ( - id text NOT NULL, - issue_id text NOT NULL, - event_type text NOT NULL, - actor text NOT NULL, - old_value text, - new_value text, - comment text, - created_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - PRIMARY KEY (id), - CONSTRAINT fk_events_issue FOREIGN KEY (issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE -); - -CREATE INDEX IF NOT EXISTS idx_events_created_at ON events (created_at); -CREATE INDEX IF NOT EXISTS idx_events_issue ON events (issue_id); - --- ============================================================ wisp_events - -CREATE TABLE IF NOT EXISTS wisp_events ( - id text NOT NULL, - issue_id text NOT NULL, - event_type text NOT NULL, - actor text DEFAULT '', - old_value text DEFAULT '', - new_value text DEFAULT '', - comment text DEFAULT '', - created_at timestamp(0) DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - PRIMARY KEY (id), - CONSTRAINT fk_wisp_events_issue FOREIGN KEY (issue_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE -); - -CREATE INDEX IF NOT EXISTS idx_wisp_events_created_at ON wisp_events (created_at); -CREATE INDEX IF NOT EXISTS idx_wisp_events_issue ON wisp_events (issue_id); - --- ============================================================ config - -CREATE TABLE IF NOT EXISTS config ( - key text NOT NULL, - value text NOT NULL, - PRIMARY KEY (key) -); - --- ============================================================ metadata - -CREATE TABLE IF NOT EXISTS metadata ( - key text NOT NULL, - value text NOT NULL, - PRIMARY KEY (key) -); - --- ============================================================ local_metadata - -CREATE TABLE IF NOT EXISTS local_metadata ( - key text NOT NULL, - value text NOT NULL DEFAULT '', - PRIMARY KEY (key) -); - --- ============================================================ issue_counter - -CREATE TABLE IF NOT EXISTS issue_counter ( - prefix text NOT NULL, - last_id integer NOT NULL DEFAULT 0, - PRIMARY KEY (prefix) -); - --- ============================================================ child_counters - -CREATE TABLE IF NOT EXISTS child_counters ( - parent_id text NOT NULL, - last_child integer NOT NULL DEFAULT 0, - PRIMARY KEY (parent_id), - CONSTRAINT fk_counter_parent FOREIGN KEY (parent_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE -); - --- ============================================================ wisp_child_counters - -CREATE TABLE IF NOT EXISTS wisp_child_counters ( - parent_id text NOT NULL, - last_child integer NOT NULL DEFAULT 0, - PRIMARY KEY (parent_id), - CONSTRAINT fk_wisp_child_counters_parent FOREIGN KEY (parent_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE -); - --- ============================================================ custom_statuses - -CREATE TABLE IF NOT EXISTS custom_statuses ( - name text NOT NULL, - category text NOT NULL DEFAULT 'unspecified', - PRIMARY KEY (name) -); - --- ============================================================ custom_types - -CREATE TABLE IF NOT EXISTS custom_types ( - name text NOT NULL, - PRIMARY KEY (name) -); - --- ============================================================ repo_mtimes - -CREATE TABLE IF NOT EXISTS repo_mtimes ( - repo_path text NOT NULL, - jsonl_path text NOT NULL, - mtime_ns bigint NOT NULL, - last_checked timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), - PRIMARY KEY (repo_path) -); - -CREATE INDEX IF NOT EXISTS idx_repo_mtimes_checked ON repo_mtimes (last_checked); -` - -// schemaVersion is stamped into the metadata table by InitSchema. A workspace -// carrying a different stamp is rejected: this proof-wedge ships no migrator. -const schemaVersion = "1" - -// schemaVersionKey lives in the metadata table (NOT config): config rows are -// surfaced by issueops.GetAllConfig / `bd config list`, so a version key there -// would leak a permanent diff against the Dolt oracle in differential runs. The -// metadata table is where the shared layer already keeps backend-internal keys. -const schemaVersionKey = "pg_schema_version" - -// schemaNameRe restricts schema names to a plain SQL identifier so they can be -// safely interpolated into CREATE SCHEMA (which cannot take a placeholder). -var schemaNameRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) - -// InitSchema creates the per-workspace schema if absent, then applies the -// embedded DDL one statement at a time and stamps the schema version. -// -// db must be a RAW *sql.DB (pgdialect.OpenRaw), not the translating one: the DDL -// is already native Postgres and routing it through Translate mangles the -// $$-quoted function bodies and rewrites now(). Version-stamp queries below use -// native $n placeholders for the same reason. -// -// All work runs on a single connection whose search_path is pinned to the target -// schema, so DDL never silently lands in public when db is handed unpinned. The -// schema name is validated (not parameterizable in DDL) and every DDL statement -// is executed on its own, since pgx's extended protocol rejects multiple -// statements in a single Exec. -func InitSchema(ctx context.Context, db *sql.DB, schema string) error { - if !schemaNameRe.MatchString(schema) { - return fmt.Errorf("postgres: invalid schema name %q", schema) - } - conn, err := db.Conn(ctx) - if err != nil { - return fmt.Errorf("postgres: acquire connection: %w", err) - } - defer conn.Close() - - // InitSchema runs on EVERY open and re-applies idempotent DDL (CREATE OR - // REPLACE FUNCTION, CREATE INDEX IF NOT EXISTS, ...). Idempotent is not the - // same as concurrency-safe: concurrent opens rewriting the same pg catalog - // tuples collide with "tuple concurrently updated" (SQLSTATE XX000). When a - // gc controller drives this store, the dispatcher, session reconciler, and - // agent subprocesses open bd concurrently, so the race is routine and it - // surfaces as spurious bd failures (e.g. the reconciler's assigned-work probe - // failing and orphan-draining live sessions). Serialize DDL application per - // schema with a session advisory lock so only one initializer touches the - // catalog at a time; distinct schemas hash to distinct keys and never block - // each other. Held on this connection and released before it returns to the - // pool. - lockKey := schemaAdvisoryLockKey(schema) - if _, err := conn.ExecContext(ctx, `SELECT pg_advisory_lock($1)`, lockKey); err != nil { - return fmt.Errorf("postgres: acquire schema-init advisory lock: %w", err) - } - // The unlock must run even when ctx is already canceled: with a done ctx, - // database/sql returns before touching the driver, the connection goes back - // to the pool healthy, and its backend session keeps holding the advisory - // lock — wedging every later open of this schema. WithoutCancel guarantees - // the unlock is attempted; if it fails on the wire instead, the driver marks - // the connection bad, the backend session dies with it, and Postgres releases - // the lock anyway, so the error is safe to drop. - defer func() { - _, _ = conn.ExecContext(context.WithoutCancel(ctx), `SELECT pg_advisory_unlock($1)`, lockKey) - }() - - if _, err := conn.ExecContext(ctx, `CREATE SCHEMA IF NOT EXISTS "`+schema+`"`); err != nil { - return fmt.Errorf("postgres: create schema %q: %w", schema, err) - } - if _, err := conn.ExecContext(ctx, `SET search_path TO "`+schema+`"`); err != nil { - return fmt.Errorf("postgres: set search_path %q: %w", schema, err) - } - for _, stmt := range splitStatements(ddl) { - if _, err := conn.ExecContext(ctx, stmt); err != nil { - return fmt.Errorf("postgres: exec DDL statement: %w\n-- statement:\n%s", err, stmt) - } - } - fresh, err := stampSchemaVersion(ctx, conn) - if err != nil { - return err - } - // Seed the default config rows ONLY on first provision — exactly like Dolt's - // migration 0016, which runs once. InitSchema runs on EVERY open (NewFromConfig → - // Provision → InitSchema) for idempotency, so seeding unconditionally would - // resurrect any config key a user has `config unset`, diverging from Dolt. - if fresh { - if err := seedDefaultConfig(ctx, conn); err != nil { - return err - } - } - return nil -} - -// defaultConfigSeeds mirrors internal/storage/schema/migrations/0016_default_config.up.sql -// exactly — the config rows a fresh Dolt workspace materializes on init. Seeding the -// identical set (and no more) keeps `bd config list` byte-identical to the Dolt oracle; -// on Dolt these land via a migration, which the wedge does not run, so InitSchema seeds -// them directly. issue_prefix is intentionally absent: it is workspace-specific and the -// caller (bd init) seeds it via store.SetConfig, matching the Dolt path. -var defaultConfigSeeds = [][2]string{ - {"compaction_enabled", "false"}, - {"compact_tier1_days", "30"}, - {"compact_tier1_dep_levels", "2"}, - {"compact_tier2_days", "90"}, - {"compact_tier2_dep_levels", "5"}, - {"compact_tier2_commits", "100"}, - {"compact_batch_size", "50"}, - {"compact_parallel_workers", "5"}, - {"auto_compact_enabled", "false"}, -} - -// seedDefaultConfig inserts the migration-0016 default config rows using -// ON CONFLICT DO NOTHING (the Postgres equivalent of the migration's INSERT IGNORE). -// InitSchema calls it only on first provision; the ON CONFLICT guard additionally -// makes it safe against a concurrent first init. Runs on the search_path-pinned -// connection so it targets the workspace config table. -func seedDefaultConfig(ctx context.Context, conn *sql.Conn) error { - for _, kv := range defaultConfigSeeds { - if _, err := conn.ExecContext(ctx, - `INSERT INTO config (key, value) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, - kv[0], kv[1]); err != nil { - return fmt.Errorf("postgres: seed default config %q: %w", kv[0], err) - } - } - return nil -} - -// stampSchemaVersion records schemaVersion in the metadata table on first init -// and refuses to open a workspace written by a binary with a different schema -// version (the proof-wedge has no migrator). It returns fresh=true only when it -// stamped a brand-new schema (no version row yet) — the signal InitSchema uses to -// seed one-time data exactly once. Runs on the search_path-pinned connection so it -// resolves the workspace metadata table. -func stampSchemaVersion(ctx context.Context, conn *sql.Conn) (fresh bool, err error) { - var stored string - err = conn.QueryRowContext(ctx, `SELECT value FROM metadata WHERE key = $1`, schemaVersionKey).Scan(&stored) - switch { - case errors.Is(err, sql.ErrNoRows): - // Provision runs on every open, so concurrent first opens of the same - // workspace (e.g. the hosted server from two goroutines) can both land - // here. ON CONFLICT DO NOTHING lets both succeed; only the opener that - // actually inserted the row (RowsAffected==1) is fresh and seeds defaults. - res, ierr := conn.ExecContext(ctx, `INSERT INTO metadata (key, value) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, schemaVersionKey, schemaVersion) - if ierr != nil { - return false, fmt.Errorf("postgres: stamp schema version: %w", ierr) - } - if n, _ := res.RowsAffected(); n == 1 { - return true, nil - } - // Lost the race: re-read what the winner stored and version-check it. - if rerr := conn.QueryRowContext(ctx, `SELECT value FROM metadata WHERE key = $1`, schemaVersionKey).Scan(&stored); rerr != nil { - return false, fmt.Errorf("postgres: read schema version after conflict: %w", rerr) - } - if stored != schemaVersion { - return false, fmt.Errorf("postgres: workspace schema version %s, this binary requires %s — no migrator in the proof-wedge, recreate the workspace or use a matching binary", stored, schemaVersion) - } - return false, nil - case err != nil: - return false, fmt.Errorf("postgres: read schema version: %w", err) - case stored != schemaVersion: - return false, fmt.Errorf("postgres: workspace schema version %s, this binary requires %s — no migrator in the proof-wedge, recreate the workspace or use a matching binary", stored, schemaVersion) - default: - return false, nil - } -} - -// splitStatements splits the DDL into individual statements on top-level ';' -// boundaries. It ignores ';' that appears inside a $$-delimited body (the -// trigger function), inside a single-quoted string literal, or inside a '--' -// line comment; line comments are dropped so a ';' in prose never terminates a -// statement. Blank fragments are omitted. -func splitStatements(ddl string) []string { - var stmts []string - var buf strings.Builder - var inString, inDollar bool - n := len(ddl) - for i := 0; i < n; i++ { - c := ddl[i] - switch { - case inString: - buf.WriteByte(c) - if c == '\'' { - if i+1 < n && ddl[i+1] == '\'' { - buf.WriteByte(ddl[i+1]) - i++ - continue - } - inString = false - } - case inDollar: - if c == '$' && i+1 < n && ddl[i+1] == '$' { - buf.WriteString("$$") - inDollar = false - i++ - continue - } - buf.WriteByte(c) - default: - switch { - case c == '-' && i+1 < n && ddl[i+1] == '-': - for i < n && ddl[i] != '\n' { - i++ - } - // Leave i at the newline (or end); the loop's i++ advances past it. - case c == '$' && i+1 < n && ddl[i+1] == '$': - buf.WriteString("$$") - inDollar = true - i++ - case c == '\'': - inString = true - buf.WriteByte(c) - case c == ';': - if s := strings.TrimSpace(buf.String()); s != "" { - stmts = append(stmts, s) - } - buf.Reset() - default: - buf.WriteByte(c) - } - } - } - if s := strings.TrimSpace(buf.String()); s != "" { - stmts = append(stmts, s) - } - return stmts -} diff --git a/internal/storage/postgres/schema_test.go b/internal/storage/postgres/schema_test.go deleted file mode 100644 index 20f1abc34..000000000 --- a/internal/storage/postgres/schema_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package postgres - -import ( - "context" - "database/sql" - "fmt" - "os" - "sync" - "testing" - "time" - - "github.com/steveyegge/beads/internal/storage/pgdialect" -) - -// TestInitSchema applies the embedded DDL into a throwaway schema on a live -// Postgres and asserts the core tables materialize. It is gated on -// BEADS_PG_TEST_URL (a pgx-parseable DSN) and skips otherwise. -func TestInitSchema(t *testing.T) { - url := os.Getenv("BEADS_PG_TEST_URL") - if url == "" { - t.Skip("BEADS_PG_TEST_URL not set; skipping Postgres schema test") - } - - schema := fmt.Sprintf("bd_schema_test_%d", time.Now().UnixNano()) - - // DDL runs over a RAW (non-translating) DB; assertions below use ? bindings - // and so need the translating DB. - raw, err := pgdialect.OpenRaw(url, schema) - if err != nil { - t.Fatalf("openraw: %v", err) - } - db, err := pgdialect.Open(url, schema) - if err != nil { - t.Fatalf("open: %v", err) - } - // Registered first so they run last (cleanups are LIFO): the schema drop - // below still has open connections when it runs. - t.Cleanup(func() { _ = raw.Close() }) - t.Cleanup(func() { _ = db.Close() }) - - ctx := context.Background() - - if err := InitSchema(ctx, raw, schema); err != nil { - t.Fatalf("InitSchema: %v", err) - } - t.Cleanup(func() { - if _, err := db.ExecContext(ctx, `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { - t.Errorf("drop schema %q: %v", schema, err) - } - }) - - for _, table := range []string{"issues", "dependencies"} { - if !tableExists(ctx, t, db, schema, table) { - t.Errorf("expected table %q to exist in schema %q", table, schema) - } - } -} - -func tableExists(ctx context.Context, t *testing.T, db *sql.DB, schema, table string) bool { - t.Helper() - var count int - err := db.QueryRowContext(ctx, - `SELECT count(*) FROM information_schema.tables WHERE table_schema = ? AND table_name = ?`, - schema, table).Scan(&count) - if err != nil { - t.Fatalf("query information_schema.tables for %q: %v", table, err) - } - return count > 0 -} - -// TestSchemaAdvisoryLockKey pins the advisory-lock key derivation. The key is a -// cross-process contract: every bd binary that opens the same workspace schema -// must derive the same key, or their InitSchema calls stop serializing against -// each other. The golden values fail loudly if the hash, prefix, or truncation -// ever changes — such a change would silently break lock interop with -// already-deployed binaries during a rolling upgrade. -func TestSchemaAdvisoryLockKey(t *testing.T) { - golden := map[string]int64{ - "beads": -7207352757236036547, - "public": -3420031902527778135, - } - for schema, want := range golden { - if got := schemaAdvisoryLockKey(schema); got != want { - t.Errorf("schemaAdvisoryLockKey(%q) = %d, want %d — changing the derivation breaks locking against deployed binaries", schema, got, want) - } - } - if a, b := schemaAdvisoryLockKey("workspace_a"), schemaAdvisoryLockKey("workspace_b"); a == b { - t.Errorf("distinct schemas derived the same advisory-lock key %d; per-workspace inits must not serialize against each other", a) - } -} - -// TestInitSchemaConcurrentOpens is the regression test for the catalog race the -// advisory lock fixes: InitSchema runs on EVERY open and re-applies idempotent -// DDL, and bd is a short-lived CLI, so a busy deployment runs many concurrent -// opens. Without per-schema serialization those opens rewrite the same pg -// catalog tuples and fail with "tuple concurrently updated" (SQLSTATE XX000). -// Each worker gets its own *sql.DB pool — its own backend session — to mimic -// separate bd processes, and all hammer InitSchema on the same fresh schema. -// Gated on BEADS_PG_TEST_URL like TestInitSchema. -func TestInitSchemaConcurrentOpens(t *testing.T) { - url := os.Getenv("BEADS_PG_TEST_URL") - if url == "" { - t.Skip("BEADS_PG_TEST_URL not set; skipping Postgres schema test") - } - - schema := fmt.Sprintf("bd_schema_race_%d", time.Now().UnixNano()) - ctx := context.Background() - - const workers = 8 - const opensPerWorker = 3 - - dbs := make([]*sql.DB, workers) - for i := range dbs { - raw, err := pgdialect.OpenRaw(url, schema) - if err != nil { - t.Fatalf("openraw %d: %v", i, err) - } - dbs[i] = raw - } - // Registered first so it runs last (cleanups are LIFO): the schema drop - // below still has open pools when it runs. - t.Cleanup(func() { - for _, db := range dbs { - _ = db.Close() - } - }) - t.Cleanup(func() { - if _, err := dbs[0].ExecContext(ctx, `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { - t.Errorf("drop schema %q: %v", schema, err) - } - }) - - start := make(chan struct{}) - errs := make(chan error, workers) - var wg sync.WaitGroup - for i := 0; i < workers; i++ { - wg.Add(1) - go func(db *sql.DB) { - defer wg.Done() - <-start - for j := 0; j < opensPerWorker; j++ { - if err := InitSchema(ctx, db, schema); err != nil { - errs <- err - return - } - } - }(dbs[i]) - } - close(start) - wg.Wait() - close(errs) - for err := range errs { - t.Errorf("concurrent InitSchema: %v", err) - } -} diff --git a/internal/storage/postgres/seed_test.go b/internal/storage/postgres/seed_test.go deleted file mode 100644 index a82fe1bc6..000000000 --- a/internal/storage/postgres/seed_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package postgres - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/steveyegge/beads/internal/storage/pgdialect" -) - -// TestSeedOnlyOnFirstProvision is the config_unset_seeded_roundtrip regression: the -// bts-rs differential oracle caught that seeding default config on EVERY open (InitSchema -// runs per NewFromConfig/Provision) resurrected any key a user had `config unset`. Seeding -// must happen only on the first provision, like Dolt's one-shot migration 0016. -func TestSeedOnlyOnFirstProvision(t *testing.T) { - url := os.Getenv("BEADS_PG_TEST_URL") - if url == "" { - t.Skip("BEADS_PG_TEST_URL not set; skipping live Postgres seed test") - } - schema := fmt.Sprintf("seedonce_%d", time.Now().UnixNano()) - ctx := context.Background() - t.Cleanup(func() { - if raw, err := pgdialect.OpenRaw(url, "public"); err == nil { - _, _ = raw.ExecContext(context.Background(), fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) - _ = raw.Close() - } - }) - - st, err := Provision(ctx, url, schema) - if err != nil { - t.Fatalf("Provision: %v", err) - } - // First provision seeds the default config rows. - if v, _ := st.GetConfig(ctx, "auto_compact_enabled"); v != "false" { - t.Fatalf("first provision: want seeded auto_compact_enabled=false, got %q", v) - } - // User unsets a seeded key. - if err := st.DeleteConfig(ctx, "auto_compact_enabled"); err != nil { - t.Fatalf("DeleteConfig: %v", err) - } - if v, _ := st.GetConfig(ctx, "auto_compact_enabled"); v != "" { - t.Fatalf("after unset: want empty, got %q", v) - } - _ = st.Close() - - // Re-open the SAME schema — InitSchema runs again. The unset key must STAY unset. - st2, err := Provision(ctx, url, schema) - if err != nil { - t.Fatalf("re-Provision: %v", err) - } - defer st2.Close() - if v, _ := st2.GetConfig(ctx, "auto_compact_enabled"); v != "" { - t.Fatalf("REGRESSION: re-provision resurrected an unset key: auto_compact_enabled=%q (want empty)", v) - } -} diff --git a/internal/storage/postgres/smoke_test.go b/internal/storage/postgres/smoke_test.go deleted file mode 100644 index 065c1eb5c..000000000 --- a/internal/storage/postgres/smoke_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package postgres - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/steveyegge/beads/internal/storage/pgdialect" - "github.com/steveyegge/beads/internal/types" -) - -// TestPGSmoke drives the full stack — dialect translation + issueops delegation -// + the synchronous readiness strategy — against a live Postgres. It proves the -// wedge actually runs, not just compiles. Gated on BEADS_PG_TEST_URL. -func TestPGSmoke(t *testing.T) { - url := os.Getenv("BEADS_PG_TEST_URL") - if url == "" { - t.Skip("BEADS_PG_TEST_URL not set; skipping live Postgres smoke") - } - schema := fmt.Sprintf("smoke_%d", time.Now().UnixNano()) - ctx := context.Background() - - st, err := New(ctx, Config{DSN: url, Schema: schema}) - if err != nil { - t.Fatalf("New: %v", err) - } - defer st.Close() - // Schema DDL must run over a raw (non-translating) DB; st.DB() is the - // translating one used for the actual issueops workload below. - raw, err := pgdialect.OpenRaw(url, schema) - if err != nil { - t.Fatalf("OpenRaw: %v", err) - } - if err := InitSchema(ctx, raw, schema); err != nil { - _ = raw.Close() - t.Fatalf("InitSchema: %v", err) - } - _ = raw.Close() - t.Cleanup(func() { - _, _ = st.DB().ExecContext(context.Background(), fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) - }) - - if err := st.SetConfig(ctx, "issue_prefix", "smoke"); err != nil { - t.Fatalf("SetConfig: %v", err) - } - - mk := func(id, title string) { - iss := &types.Issue{ - ID: id, Title: title, IssueType: types.IssueType("task"), - Status: types.Status("open"), Priority: 2, CreatedBy: "tester", - } - if err := st.CreateIssue(ctx, iss, "tester"); err != nil { - t.Fatalf("CreateIssue(%s): %v", id, err) - } - } - mk("smoke-1", "first") - mk("smoke-2", "second") - - got, err := st.GetIssue(ctx, "smoke-1") - if err != nil || got == nil { - t.Fatalf("GetIssue(smoke-1): %v (got=%v)", err, got) - } - if got.Title != "first" { - t.Fatalf("GetIssue title = %q, want %q", got.Title, "first") - } - - readyIDs := func() []string { - rows, err := st.GetReadyWork(ctx, types.WorkFilter{}) - if err != nil { - t.Fatalf("GetReadyWork: %v", err) - } - ids := make([]string, len(rows)) - for i, r := range rows { - ids[i] = r.ID - } - return ids - } - - // Both open, no deps -> both ready. - if ids := readyIDs(); len(ids) != 2 { - t.Fatalf("before dep: ready=%v, want 2", ids) - } - - // smoke-2 depends on (is blocked by) smoke-1. - dep := &types.Dependency{IssueID: "smoke-2", DependsOnID: "smoke-1", Type: types.DependencyType("blocks"), CreatedBy: "tester"} - if err := st.AddDependency(ctx, dep, "tester"); err != nil { - t.Fatalf("AddDependency: %v", err) - } - - // Now only smoke-1 is ready (readiness projection ran in-tx). - if ids := readyIDs(); len(ids) != 1 || ids[0] != "smoke-1" { - t.Fatalf("after dep: ready=%v, want [smoke-1]", ids) - } - - // Close smoke-1 -> smoke-2 unblocks. - if err := st.CloseIssue(ctx, "smoke-1", "done", "tester", ""); err != nil { - t.Fatalf("CloseIssue: %v", err) - } - if ids := readyIDs(); len(ids) != 1 || ids[0] != "smoke-2" { - t.Fatalf("after close: ready=%v, want [smoke-2]", ids) - } -} diff --git a/internal/storage/postgres/store.go b/internal/storage/postgres/store.go deleted file mode 100644 index 9304023b6..000000000 --- a/internal/storage/postgres/store.go +++ /dev/null @@ -1,87 +0,0 @@ -// Package postgres is the Postgres backend for bd: the shared sqlkit.Store -// bundled with the Postgres dialect and a readiness strategy, plus the non-Dolt -// markers. It is a strategy bundle over the shared SQL-family implementation — -// no storage semantics are re-derived here. -package postgres - -import ( - "context" - - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/storage/sqlkit" -) - -// Backend is the metadata.json backend identifier for this store. -const Backend = "postgres" - -// Store is the Postgres backend. It embeds *sqlkit.Store, so it inherits every -// core method and overrides only what Postgres specializes (nothing yet — the -// readiness projection and FOR UPDATE SKIP LOCKED claim are later strategy -// swaps that override GetReadyWork/ClaimReadyIssue without touching sqlkit). -type Store struct { - *sqlkit.Store - // unsupportedDoltStorage (generated, unsupported_gen.go) provides the - // capability surface Postgres does not implement — VC/history/remote/sync/ - // compaction/federation and a few non-demo core methods — each returning a - // typed *storage.ErrUnsupported. It is generated as the exact complement of - // *sqlkit.Store's method set, so there is no ambiguous-selector overlap. - unsupportedDoltStorage -} - -// Compile-time guard on the generated skip list: the composite backend (real -// *sqlkit.Store methods plus the typed-unsupported shell) must fully satisfy -// storage.DoltStorage. This fails the build on skip-list drift — a skipped -// method sqlkit does not actually implement (missing method), or a method both -// implemented and stubbed (ambiguous selector) — so no method can silently -// become "unsupported". -var _ storage.DoltStorage = (*Store)(nil) - -// Config configures a Postgres backend. -type Config struct { - DSN string // password merged from env at open; never persisted to metadata.json - Schema string // per-workspace schema, pinned via search_path -} - -// New builds the Postgres backend from its strategy bundle: the pg dialect -// (translating driver) and a no-op readiness strategy (the shared issueops -// helpers already maintain is_blocked targeted and in-transaction, matching the -// Dolt reference; SyncReadiness would only add a redundant per-write recompute). -func New(ctx context.Context, cfg Config) (*Store, error) { - base, err := sqlkit.New(ctx, sqlkit.Config{ - Dialect: pgDialect{dsn: cfg.DSN, searchPath: cfg.Schema}, - Readiness: sqlkit.NoopReadiness{}, - }) - if err != nil { - return nil, err - } - return &Store{Store: base}, nil -} - -// CommitGraphUnsupported is the negative maintenance marker (see -// storage.NonCommitGraphBackend): it tells cmd/bd's PostRun to skip Dolt-only -// maintenance. Only non-Dolt backends implement it, so the default path is -// unaffected. -func (s *Store) CommitGraphUnsupported() bool { return true } - -// CommitPending is a no-op on a non-version-controlled backend; write paths -// invoke it opportunistically and must not error. -func (s *Store) CommitPending(_ context.Context, _ string) (bool, error) { - return false, nil -} - -// Commit and its variants are no-ops on Postgres, NOT unsupported: a Dolt -// "commit" flushes the working set to history, but every sqlkit write already -// committed its own SQL transaction, so the data is durable. Command paths call -// store.Commit() opportunistically after writes; returning nil lets those writes -// stand instead of erroring on a history operation that has no PG meaning. -func (s *Store) Commit(_ context.Context, _ string) error { return nil } -func (s *Store) CommitWithConfig(_ context.Context, _ string) error { return nil } -func (s *Store) CommitMergeResolution(_ context.Context, _ string) error { return nil } - -// Compile-time proof the Postgres backend satisfies the full storage seam -// (core methods via *sqlkit.Store, the rest via the generated shell) and the -// negative maintenance marker. -var ( - _ storage.DoltStorage = (*Store)(nil) - _ storage.NonCommitGraphBackend = (*Store)(nil) -) diff --git a/internal/storage/postgres/unsupported.go b/internal/storage/postgres/unsupported.go deleted file mode 100644 index 6d1e45ae8..000000000 --- a/internal/storage/postgres/unsupported.go +++ /dev/null @@ -1,24 +0,0 @@ -package postgres - -import ( - "github.com/steveyegge/beads/internal/storage" -) - -// Regeneration: the postgres shell is the exact complement of *sqlkit.Store's -// method set. Regenerate unsupported_gen.go with the directive below after -// finalizing the skip list (the sqlkit-implemented ops — the integrator fills -// in ). gen's strict unmatched-skip validation then doubles as a drift -// tripwire against DoltStorage interface changes. -// -//go:generate go run ../unsupportedgen -pkg postgres -src .. -out unsupported_gen.go -type DoltStorage -skip AddDependency,AddIssueComment,AddLabel,ClaimIssue,ClaimReadyIssue,Close,CloseIssue,Commit,CommitMergeResolution,CommitPending,CommitWithConfig,CountDependencies,CountDependents,CountEvents,CountIssueComments,CountIssues,CountIssuesByGroup,CreateIssue,CreateIssues,DeleteConfig,DeleteIssue,DeleteIssues,DetectCycles,GetAllConfig,GetBlockedIssues,GetBlockingInfoForIssues,GetCommentCounts,GetCommentsForIssues,GetConfig,GetCustomStatuses,GetCustomStatusesDetailed,GetCustomTypes,GetDependencies,GetDependenciesWithMetadata,GetDependencyCounts,GetDependencyRecords,GetDependencyRecordsForIssues,GetDependencyTree,GetDependents,GetDependentsWithMetadata,GetEpicsEligibleForClosure,GetEvents,GetInfraTypes,GetIssue,GetIssueByExternalRef,GetIssueComments,GetIssuesByIDs,GetIssuesByLabel,GetLabels,GetLabelsForIssues,GetLocalMetadata,GetMetadata,GetNewlyUnblockedByClose,GetNextChildID,GetReadyWork,GetReadyWorkWithCounts,GetStaleIssues,GetStatistics,IsBlocked,IsInfraTypeCtx,IterBlockedIssues,IterDependenciesWithMetadata,IterDependentsWithMetadata,IterEvents,IterIssueComments,IterIssues,IterReadyWork,IterWisps,ListWisps,RemoveDependency,RemoveLabel,ReopenIssue,RunInTransaction,SearchIssues,SearchIssuesWithCounts,SetConfig,SetLocalMetadata,SetMetadata,UpdateIssue,UpdateIssueType,AddComment,ClearRepoMtime,CountDependentsByStatus,DeleteIssuesBySourceRepo,FindWispDependentsRecursive,GetAllDependencyRecords,GetAllEventsSince,GetMoleculeLastActivity,GetMoleculeProgress,GetRepoMtime,ImportIssueComment,IterAllDependencyRecords,IterAllEventsSince,PromoteFromEphemeral,SetRepoMtime,UpdateIssueID,CreateIssuesWithFullOptions,SlotClear,SlotGet,SlotSet,SearchIssueIDs,HeartbeatIssue,ReclaimExpiredLeases,UnclaimIssue - -// errUnsupported is the constructor every generated stub in unsupported_gen.go -// calls. It returns the backend-agnostic *storage.ErrUnsupported sentinel, whose -// Error() renders `operation %q not supported by the %s backend`. That message is -// correct for every stubbed op — the shell also stubs non-VC core methods -// (GetStatistics, DeleteIssues, ...) that a Dolt backend is not the remedy for — -// so it carries no history/version-control hint. Backend is the package const -// "postgres"; callers can errors.As/Is on the returned sentinel. -func errUnsupported(op string) error { - return &storage.ErrUnsupported{Op: op, Backend: Backend} -} diff --git a/internal/storage/postgres/unsupported_gen.go b/internal/storage/postgres/unsupported_gen.go deleted file mode 100644 index ce228433e..000000000 --- a/internal/storage/postgres/unsupported_gen.go +++ /dev/null @@ -1,236 +0,0 @@ -// Code generated by gen (postgres typed-unsupported shell); DO NOT EDIT. - -package postgres - -import ( - "context" - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/types" -) - -// unsupportedDoltStorage is the generated typed-unsupported shell for storage.DoltStorage. -// Every method returns *storage.ErrUnsupported; embed it and override the -// real slice. DO NOT hand-edit — regenerate with `go generate ./...`. -type unsupportedDoltStorage struct{} - -func (unsupportedDoltStorage) MergeSlotCreate(_ context.Context, _ string) (_ *types.Issue, err error) { - err = errUnsupported("MergeSlotCreate") - return -} - -func (unsupportedDoltStorage) MergeSlotCheck(_ context.Context) (_ *storage.MergeSlotStatus, err error) { - err = errUnsupported("MergeSlotCheck") - return -} - -func (unsupportedDoltStorage) MergeSlotAcquire(_ context.Context, _ string, _ string, _ bool) (_ *storage.MergeSlotResult, err error) { - err = errUnsupported("MergeSlotAcquire") - return -} - -func (unsupportedDoltStorage) MergeSlotRelease(_ context.Context, _ string, _ string) (err error) { - err = errUnsupported("MergeSlotRelease") - return -} - -func (unsupportedDoltStorage) Branch(_ context.Context, _ string) (err error) { - err = errUnsupported("Branch") - return -} - -func (unsupportedDoltStorage) Checkout(_ context.Context, _ string) (err error) { - err = errUnsupported("Checkout") - return -} - -func (unsupportedDoltStorage) CurrentBranch(_ context.Context) (_ string, err error) { - err = errUnsupported("CurrentBranch") - return -} - -func (unsupportedDoltStorage) DeleteBranch(_ context.Context, _ string) (err error) { - err = errUnsupported("DeleteBranch") - return -} - -func (unsupportedDoltStorage) ListBranches(_ context.Context) (_ []string, err error) { - err = errUnsupported("ListBranches") - return -} - -func (unsupportedDoltStorage) CommitExists(_ context.Context, _ string) (_ bool, err error) { - err = errUnsupported("CommitExists") - return -} - -func (unsupportedDoltStorage) GetCurrentCommit(_ context.Context) (_ string, err error) { - err = errUnsupported("GetCurrentCommit") - return -} - -func (unsupportedDoltStorage) Status(_ context.Context) (_ *storage.Status, err error) { - err = errUnsupported("Status") - return -} - -func (unsupportedDoltStorage) Log(_ context.Context, _ int) (_ []storage.CommitInfo, err error) { - err = errUnsupported("Log") - return -} - -func (unsupportedDoltStorage) Merge(_ context.Context, _ string) (_ []storage.Conflict, err error) { - err = errUnsupported("Merge") - return -} - -func (unsupportedDoltStorage) GetConflicts(_ context.Context) (_ []storage.Conflict, err error) { - err = errUnsupported("GetConflicts") - return -} - -func (unsupportedDoltStorage) ResolveConflicts(_ context.Context, _ string, _ string) (err error) { - err = errUnsupported("ResolveConflicts") - return -} - -func (unsupportedDoltStorage) History(_ context.Context, _ string) (_ []*storage.HistoryEntry, err error) { - err = errUnsupported("History") - return -} - -func (unsupportedDoltStorage) AsOf(_ context.Context, _ string, _ string) (_ *types.Issue, err error) { - err = errUnsupported("AsOf") - return -} - -func (unsupportedDoltStorage) Diff(_ context.Context, _ string, _ string) (_ []*storage.DiffEntry, err error) { - err = errUnsupported("Diff") - return -} - -func (unsupportedDoltStorage) AddRemote(_ context.Context, _ string, _ string) (err error) { - err = errUnsupported("AddRemote") - return -} - -func (unsupportedDoltStorage) RemoveRemote(_ context.Context, _ string) (err error) { - err = errUnsupported("RemoveRemote") - return -} - -func (unsupportedDoltStorage) HasRemote(_ context.Context, _ string) (_ bool, err error) { - err = errUnsupported("HasRemote") - return -} - -func (unsupportedDoltStorage) ListRemotes(_ context.Context) (_ []storage.RemoteInfo, err error) { - err = errUnsupported("ListRemotes") - return -} - -func (unsupportedDoltStorage) Push(_ context.Context) (err error) { - err = errUnsupported("Push") - return -} - -func (unsupportedDoltStorage) Pull(_ context.Context) (err error) { - err = errUnsupported("Pull") - return -} - -func (unsupportedDoltStorage) ForcePush(_ context.Context) (err error) { - err = errUnsupported("ForcePush") - return -} - -func (unsupportedDoltStorage) PushRemote(_ context.Context, _ string, _ bool) (err error) { - err = errUnsupported("PushRemote") - return -} - -func (unsupportedDoltStorage) PullRemote(_ context.Context, _ string) (err error) { - err = errUnsupported("PullRemote") - return -} - -func (unsupportedDoltStorage) Fetch(_ context.Context, _ string) (err error) { - err = errUnsupported("Fetch") - return -} - -func (unsupportedDoltStorage) PushTo(_ context.Context, _ string) (err error) { - err = errUnsupported("PushTo") - return -} - -func (unsupportedDoltStorage) PullFrom(_ context.Context, _ string) (_ []storage.Conflict, err error) { - err = errUnsupported("PullFrom") - return -} - -func (unsupportedDoltStorage) Sync(_ context.Context, _ string, _ string) (_ *storage.SyncResult, err error) { - err = errUnsupported("Sync") - return -} - -func (unsupportedDoltStorage) SyncStatus(_ context.Context, _ string) (_ *storage.SyncStatus, err error) { - err = errUnsupported("SyncStatus") - return -} - -func (unsupportedDoltStorage) AddFederationPeer(_ context.Context, _ *storage.FederationPeer) (err error) { - err = errUnsupported("AddFederationPeer") - return -} - -func (unsupportedDoltStorage) GetFederationPeer(_ context.Context, _ string) (_ *storage.FederationPeer, err error) { - err = errUnsupported("GetFederationPeer") - return -} - -func (unsupportedDoltStorage) ListFederationPeers(_ context.Context) (_ []*storage.FederationPeer, err error) { - err = errUnsupported("ListFederationPeers") - return -} - -func (unsupportedDoltStorage) RemoveFederationPeer(_ context.Context, _ string) (err error) { - err = errUnsupported("RemoveFederationPeer") - return -} - -func (unsupportedDoltStorage) CheckEligibility(_ context.Context, _ string, _ int) (_ bool, _ string, err error) { - err = errUnsupported("CheckEligibility") - return -} - -func (unsupportedDoltStorage) ApplyCompaction(_ context.Context, _ string, _ int, _ int, _ int, _ string) (err error) { - err = errUnsupported("ApplyCompaction") - return -} - -func (unsupportedDoltStorage) GetTier1Candidates(_ context.Context) (_ []*types.CompactionCandidate, err error) { - err = errUnsupported("GetTier1Candidates") - return -} - -func (unsupportedDoltStorage) GetTier2Candidates(_ context.Context) (_ []*types.CompactionCandidate, err error) { - err = errUnsupported("GetTier2Candidates") - return -} - -func (unsupportedDoltStorage) SnapshotIssue(_ context.Context, _ string, _ int) (err error) { - err = errUnsupported("SnapshotIssue") - return -} - -func (unsupportedDoltStorage) GetCompactionSnapshot(_ context.Context, _ string) (_ *types.IssueSnapshot, err error) { - err = errUnsupported("GetCompactionSnapshot") - return -} - -func (unsupportedDoltStorage) RestoreFromSnapshot(_ context.Context, _ string) (_ *types.IssueSnapshot, err error) { - err = errUnsupported("RestoreFromSnapshot") - return -} - -// NOTE: partial shell (104 methods skipped); the embedding composite must assert storage.DoltStorage itself. diff --git a/internal/storage/sqlbuild/sort.go b/internal/storage/sqlbuild/sort.go index 53bfd7825..755b7b576 100644 --- a/internal/storage/sqlbuild/sort.go +++ b/internal/storage/sqlbuild/sort.go @@ -70,11 +70,9 @@ func OrderByForColumns(sortBy string, sortDesc bool, col func(sortKey string) st if sortBy == "" || sortBy == "priority" { return fmt.Sprintf("ORDER BY %s %s, %s DESC, %s ASC", col("priority"), dir, col("created"), col("id")) } - // A nullable sort column (closed_at, assignee) must order NULLs the MySQL way — NULL - // treated as lowest, so first on ASC and last on DESC — on every backend. MySQL/Dolt/ - // SQLite do this by default; Postgres defaults the opposite way (NULLS FIRST on DESC), - // so lead with an explicit (col IS NULL) key for those two. NOT NULL columns keep the - // plain clause (byte-identical to before). + // A nullable sort column (closed_at, assignee) treats NULL as lowest: first on ASC + // and last on DESC. Lead with an explicit (col IS NULL) key so the contract does not + // depend on a driver's default NULL ordering. NOT NULL columns keep the plain clause. if sortBy == "closed" || sortBy == "assignee" { return fmt.Sprintf("ORDER BY (%s IS NULL) %s, %s %s, %s ASC", col(sortBy), flipDir(dir), col(sortBy), dir, col("id")) } @@ -102,7 +100,7 @@ func OrderBy(sortBy string, sortDesc bool, table string) string { // Less is the Go-side mirror of OrderBy for merge sorts over rows fetched // from separate queries (issues + wisps). It must order exactly the way the -// SQL does, including MySQL NULL-first semantics for nullable columns; +// SQL does, including NULL-first ascending semantics for nullable columns; // otherwise a post-merge limit cut keeps a different row set than SQL // selected. func Less(a, b *types.Issue, sortBy string, sortDesc bool) bool { diff --git a/internal/storage/sqlite/completeness_test.go b/internal/storage/sqlite/completeness_test.go index 06af61c75..04d727634 100644 --- a/internal/storage/sqlite/completeness_test.go +++ b/internal/storage/sqlite/completeness_test.go @@ -9,24 +9,22 @@ import ( "github.com/steveyegge/beads/internal/storage/conformance" ) -// legitimatelyUnsupported is the EXPLICIT denominator of every storage.DoltStorage -// method the Postgres wedge deliberately does not implement, each with a reason. +// legitimatelyUnsupported is the explicit set of storage.DoltStorage methods that +// SQLite deliberately does not implement, each with a reason. // // The interface-completeness gate (TestInterfaceCompleteness) asserts the // generated capability shell (unsupported_gen.go) contains EXACTLY these methods // — no more, no less. This is the guard the red-team had to stand in for: a -// method that silently resolves to the typed-unsupported shell (the class that -// bit us: DeleteIssues, GetCustomStatuses, the batch reads, …) fails this test +// method that silently resolves to the typed-unsupported shell fails this test // at build time instead of failing a user at runtime. Adding a method to the // shell without an entry here is a hard error that forces the triage question: // "implement it in sqlkit, or record here why the wedge legitimately omits it." var legitimatelyUnsupported = map[string]string{ - // VersionControl — the Dolt commit graph. The wedge's whole premise is - // "bd on Postgres, history/version-control simply absent". + // VersionControl — SQLite has no Dolt commit graph. "Branch": "VC: Dolt branch", "Checkout": "VC: Dolt checkout", "CurrentBranch": "VC", "DeleteBranch": "VC", "ListBranches": "VC", - // Commit/CommitWithConfig/CommitMergeResolution are NO-OPS on Postgres (data - // is already durable per-tx), not unsupported — implemented on postgres.Store. + // Commit/CommitWithConfig/CommitMergeResolution are no-ops on SQLite because + // data is already durable per transaction; sqlite.Store implements them. "CommitExists": "VC", "GetCurrentCommit": "VC: Dolt hash", "Status": "VC", "Log": "VC: dolt_log", "Merge": "VC", "GetConflicts": "VC", "ResolveConflicts": "VC", diff --git a/internal/storage/sqlite/dsn.go b/internal/storage/sqlite/dsn.go index 50cfd9a03..672756f7d 100644 --- a/internal/storage/sqlite/dsn.go +++ b/internal/storage/sqlite/dsn.go @@ -13,9 +13,8 @@ import "strings" // inter-process contention. // - _txlock=immediate: writers take the write lock up front and fail fast rather // than deadlocking mid-transaction on lock upgrade. -// - case_sensitive_like(1): LIKE parity with the other backends. Dolt/MySQL use -// the utf8mb4_0900_bin table collation and Postgres uses collation "C", so -// `id LIKE 'bd-A%'` is case-sensitive there; SQLite's default LIKE is +// - case_sensitive_like(1): LIKE parity with Dolt's binary table collation. +// SQLite's default LIKE is // ASCII-case-INsensitive, which silently diverged on raw-cased operands // (--id-prefix filtering, parent-descendant `id LIKE parent || '.%'`, // GetNextChildID's child scan). Intentionally case-insensitive searches are @@ -28,14 +27,12 @@ import "strings" // counts mega-query renders (translated to strftime by sqlitedialect) returns NULL and // deps_json exposes a zero created_at instead of the real timestamp. datetime stores // "2026-07-09 12:34:56", which strftime yields as "2026-07-09T12:34:56Z", giving the -// same whole-second GRANULARITY as Dolt/MySQL datetime(0) and unifying bound timestamps +// same whole-second granularity as Dolt datetime(0) and unifying bound timestamps // with the DEFAULT CURRENT_TIMESTAMP rows into one sortable, strftime-parseable format. // -// Note the granularity matches but the sub-second RULE does not: Go's Format truncates -// the fraction, whereas Dolt/MySQL datetime(0) round half-up (Postgres keeps microseconds). -// This sub-second divergence is an accepted, documented waiver — the portable contract is -// a whole-second round-trip (see conformance.testAuditImportCommentSubSecond); forcing -// round-down parity would only discard the SQL backends' native precision. +// Note the granularity matches but the sub-second rule does not: Go's Format truncates +// the fraction, whereas Dolt datetime(0) rounds half-up. This accepted divergence is +// covered by conformance.testAuditImportCommentSubSecond. func dsn(path string) string { if strings.HasPrefix(path, "file:") { return path diff --git a/internal/storage/sqlite/schema.go b/internal/storage/sqlite/schema.go index 8087532e8..52d077b5b 100644 --- a/internal/storage/sqlite/schema.go +++ b/internal/storage/sqlite/schema.go @@ -9,7 +9,7 @@ import ( "strings" ) -// ddl is the SQLite backend schema, embedded from schema.sql. It is the Dolt/MySQL +// ddl is the SQLite backend schema, embedded from schema.sql. It is the Dolt // schema with SQLite adjustments: ENGINE/CHARSET/COLLATE stripped, inline KEY/UNIQUE // KEY lifted to CREATE INDEX (SQLite forbids inline non-PK indexes), ON UPDATE // CURRENT_TIMESTAMP dropped (unsupported), json → text, and prefix-index lengths diff --git a/internal/storage/sqlite/unsupported.go b/internal/storage/sqlite/unsupported.go index 5c92264b2..6c9971684 100644 --- a/internal/storage/sqlite/unsupported.go +++ b/internal/storage/sqlite/unsupported.go @@ -4,8 +4,8 @@ import "github.com/steveyegge/beads/internal/storage" //go:generate go run ../unsupportedgen -pkg sqlite -src .. -out unsupported_gen.go -type DoltStorage -skip AddDependency,AddIssueComment,AddLabel,ClaimIssue,ClaimReadyIssue,Close,CloseIssue,Commit,CommitMergeResolution,CommitPending,CommitWithConfig,CountDependencies,CountDependents,CountEvents,CountIssueComments,CountIssues,CountIssuesByGroup,CreateIssue,CreateIssues,DeleteConfig,DeleteIssue,DeleteIssues,DetectCycles,GetAllConfig,GetBlockedIssues,GetBlockingInfoForIssues,GetCommentCounts,GetCommentsForIssues,GetConfig,GetCustomStatuses,GetCustomStatusesDetailed,GetCustomTypes,GetDependencies,GetDependenciesWithMetadata,GetDependencyCounts,GetDependencyRecords,GetDependencyRecordsForIssues,GetDependencyTree,GetDependents,GetDependentsWithMetadata,GetEpicsEligibleForClosure,GetEvents,GetInfraTypes,GetIssue,GetIssueByExternalRef,GetIssueComments,GetIssuesByIDs,GetIssuesByLabel,GetLabels,GetLabelsForIssues,GetLocalMetadata,GetMetadata,GetNewlyUnblockedByClose,GetNextChildID,GetReadyWork,GetReadyWorkWithCounts,GetStaleIssues,GetStatistics,IsBlocked,IsInfraTypeCtx,IterBlockedIssues,IterDependenciesWithMetadata,IterDependentsWithMetadata,IterEvents,IterIssueComments,IterIssues,IterReadyWork,IterWisps,ListWisps,RemoveDependency,RemoveLabel,ReopenIssue,RunInTransaction,SearchIssues,SearchIssuesWithCounts,SetConfig,SetLocalMetadata,SetMetadata,UpdateIssue,UpdateIssueType,AddComment,ClearRepoMtime,CountDependentsByStatus,DeleteIssuesBySourceRepo,FindWispDependentsRecursive,GetAllDependencyRecords,GetAllEventsSince,GetMoleculeLastActivity,GetMoleculeProgress,GetRepoMtime,ImportIssueComment,IterAllDependencyRecords,IterAllEventsSince,PromoteFromEphemeral,SetRepoMtime,UpdateIssueID,CreateIssuesWithFullOptions,SlotClear,SlotGet,SlotSet,SearchIssueIDs,HeartbeatIssue,ReclaimExpiredLeases,UnclaimIssue -// errUnsupported returns the backend-agnostic *storage.ErrUnsupported sentinel; Backend -// is "sqlite". Same allowlist as postgres/mysql (identical sqlkit base + overrides). +// errUnsupported returns the backend-agnostic *storage.ErrUnsupported sentinel for +// operations that are specific to Dolt's version-control capabilities. func errUnsupported(op string) error { return &storage.ErrUnsupported{Op: op, Backend: Backend} } diff --git a/internal/storage/sqlitedialect/driver.go b/internal/storage/sqlitedialect/driver.go index 078ceadad..75e825732 100644 --- a/internal/storage/sqlitedialect/driver.go +++ b/internal/storage/sqlitedialect/driver.go @@ -9,8 +9,8 @@ import ( ) // Open returns a *sql.DB over modernc.org/sqlite whose Prepare translates bd's -// canonical MySQL SQL to SQLite (see Translate). Like pgdialect it exposes no -// Execer/Queryer fast path, so database/sql routes every statement through Prepare. +// canonical MySQL SQL to SQLite (see Translate). It exposes no Execer/Queryer fast +// path, so database/sql routes every statement through Prepare. // The caller supplies foreign-key enforcement etc. via DSN pragmas. func Open(dsn string) (*sql.DB, error) { return sql.OpenDB(&connector{dsn: dsn}), nil diff --git a/internal/storage/sqlitedialect/translate.go b/internal/storage/sqlitedialect/translate.go index 218360dc6..b5dd8cc39 100644 --- a/internal/storage/sqlitedialect/translate.go +++ b/internal/storage/sqlitedialect/translate.go @@ -1,11 +1,7 @@ -// Package sqlitedialect translates bd's canonical (MySQL-dialect) SQL to SQLite and -// wraps the modernc.org/sqlite driver. SQLite is closer to MySQL than Postgres is — -// backtick identifiers, `?` placeholders, bool→integer binding, INSTR, json_object, -// REPLACE INTO, and CURRENT_TIMESTAMP (UTC) all work natively — so the translation is -// a small subset of pgdialect's: INSERT IGNORE, the ON DUPLICATE KEY upsert (standard -// ON CONFLICT, shared with Postgres), CONCAT, and unqualified UPDATE SET targets. The -// string-scanning machinery (codeMask/rewriteFuncCalls/…) is shared in shape with -// pgdialect. Isolated here; shared issueops/sqlkit and other backends are untouched. +// Package sqlitedialect translates bd's canonical MySQL-dialect SQL to SQLite and +// wraps the modernc.org/sqlite driver. SQLite accepts many of the canonical forms +// natively; this package rewrites the remaining INSERT, upsert, function, and UPDATE +// forms. It stays isolated so shared issueops/sqlkit code remains backend-neutral. package sqlitedialect import ( @@ -93,7 +89,7 @@ func rewriteInsertIgnore(sql string) string { return insertIgnoreRe.ReplaceAllString(sql, "INSERT OR IGNORE INTO") } -// --- ON DUPLICATE KEY UPDATE -> ON CONFLICT (standard SQL, same syntax as Postgres) --- +// --- ON DUPLICATE KEY UPDATE -> SQLite ON CONFLICT --- // onConflictTargets maps a table to its ON CONFLICT arbiter (the key MySQL's ON // DUPLICATE KEY infers). Only DO UPDATE upserts need an entry. @@ -349,7 +345,7 @@ func scanKeyword(s string, m []bool, from int, kw string) int { return -1 } -// --- shared string-scanning machinery (shape mirrors pgdialect) --- +// --- string-scanning machinery used by the SQLite rewrites --- func codeMask(sql string) []bool { b := []byte(sql) diff --git a/internal/storage/sqlkit/queries.go b/internal/storage/sqlkit/queries.go index ba4f8be3d..b7f14d96b 100644 --- a/internal/storage/sqlkit/queries.go +++ b/internal/storage/sqlkit/queries.go @@ -58,8 +58,8 @@ func (s *Store) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFil // CountReadyWork returns the total ready-work count for filter. It is identical // to len(GetReadyWorkWithCounts(filter with Limit=0)) but sizes the total with -// cheap indexed COUNT(*)s instead of re-running the counts mega-query, so every -// SQL backend (Postgres/MySQL/SQLite) satisfies storage.ReadyWorkCounter and +// cheap indexed COUNT(*)s instead of re-running the counts mega-query, so SQLite +// satisfies storage.ReadyWorkCounter and // `bd ready --json` prints "Showing X of N" without the unbounded pass. func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (int, error) { var n int diff --git a/internal/storage/sqlkit/store.go b/internal/storage/sqlkit/store.go index 638098c13..1102ccf7d 100644 --- a/internal/storage/sqlkit/store.go +++ b/internal/storage/sqlkit/store.go @@ -58,7 +58,7 @@ func New(ctx context.Context, cfg Config) (*Store, error) { // DB exposes the underlying pool for diagnostics/migrations. func (s *Store) DB() *sql.DB { return s.db } -// DialectName reports the backend's dialect ("postgres", "sqlite", …). +// DialectName reports the backend's SQL dialect. func (s *Store) DialectName() string { return s.dialect.Name() } // Close closes the pool. Idempotent. @@ -77,9 +77,7 @@ func (s *Store) Close() error { // would queue behind any in-flight writer and fail with SQLITE_BUSY after // busy_timeout, forfeiting WAL's reader/writer independence. TxOptions.ReadOnly // suppresses beginMode (plain deferred BEGIN), letting reads run concurrently -// with a writer. On the server backends it maps to START TRANSACTION READ ONLY -// (mysql) / BEGIN READ ONLY (pgx), which every withReadTx callback satisfies: -// they are pure issueops reads. +// with a writer. Every withReadTx callback is a pure issueops read. func (s *Store) withReadTx(ctx context.Context, fn func(tx *sql.Tx) error) error { if s.closed.Load() { return ErrStoreClosed diff --git a/internal/storage/sqlkit/strategy.go b/internal/storage/sqlkit/strategy.go index 3ac34cd16..a10705bf2 100644 --- a/internal/storage/sqlkit/strategy.go +++ b/internal/storage/sqlkit/strategy.go @@ -1,12 +1,12 @@ -// Package sqlkit is the single, shared SQL-family implementation of bd's storage -// semantics. Every SQL backend (Dolt, Postgres, SQLite, MySQL, …) is a strategy -// bundle over this one implementation: +// Package sqlkit is the shared relational implementation used by the SQLite +// storage backend. It remains separated from the public storage interface as a +// strategy bundle: // // - Dialect — the SQL flavor (a translating driver or native emit) // - ReadinessStrategy — how the is_blocked projection is kept current // - ClaimStrategy — how a ready issue is atomically claimed // -// A concrete backend embeds *Store, inherits every method, and overrides only +// SQLite embeds *Store, inherits every method, and overrides only // the handful it specializes (a "method override") or adds capability-interface // methods for operations that don't exist in the core contract (e.g. Dolt // history). The query/graph/ordering/id-mint/cycle semantics live here once, in @@ -22,9 +22,8 @@ import ( ) // Dialect opens the backend's *sql.DB and names it. The shared layer emits -// canonical (MySQL-dialect) SQL; a Dialect whose driver translates that text to -// the backend's flavor is how a non-MySQL SQL engine joins the family without -// changing the shared layer (see internal/storage/pgdialect for Postgres). +// canonical MySQL-dialect SQL inherited from the Dolt implementation; the SQLite +// dialect translates that text without changing the shared layer. type Dialect interface { Name() string Open(ctx context.Context) (*sql.DB, error) diff --git a/internal/storage/unsupportedgen/gen_test.go b/internal/storage/unsupportedgen/gen_test.go index 09b6cde74..460f03381 100644 --- a/internal/storage/unsupportedgen/gen_test.go +++ b/internal/storage/unsupportedgen/gen_test.go @@ -25,7 +25,7 @@ import ( // shellBackends are the backends whose typed-unsupported shell this tool // generates. Each dir is relative to this package (internal/storage/unsupportedgen). -var shellBackends = []string{"postgres", "mysql", "sqlite"} +var shellBackends = []string{"sqlite"} func TestGeneratedShellsAreUpToDate(t *testing.T) { // storage package source, relative to this package dir (one level up). diff --git a/internal/storage/unsupportedgen/main.go b/internal/storage/unsupportedgen/main.go index 4aed4efd9..9a93e825b 100644 --- a/internal/storage/unsupportedgen/main.go +++ b/internal/storage/unsupportedgen/main.go @@ -12,7 +12,7 @@ // so the AST flatten resolves all embeds locally. // // Invoked via `//go:generate go run ../unsupportedgen -pkg -src .. ...` -// from a backend package dir (e.g. internal/storage/postgres), whose CWD makes +// from a backend package dir (e.g. internal/storage/sqlite), whose CWD makes // the storage package ".." — hence -src "..". package main @@ -65,7 +65,7 @@ func main() { // genConfig is the parameterized generation request. Its zero-flag defaults // (see defaultConfig) reproduce a backend's committed shell byte-for-byte; the // flags exist so the same generator can emit a partial shell for another -// package (e.g. postgres) that embeds a real store alongside the generated one. +// package (e.g. sqlite) that embeds a real store alongside the generated one. type genConfig struct { pkg string // -pkg : package clause of the generated file src string // -src : dir of the storage package to parse @@ -133,7 +133,7 @@ type pkgInfo struct { // runConfig is the parameterized generator. At defaultConfig() values it emits // a backend's committed shell byte-for-byte; -pkg/-type/-skip let it emit a -// partial shell for another package (e.g. postgres) that embeds a real store +// partial shell for another package (e.g. sqlite) that embeds a real store // alongside the generated one. func runConfig(cfg genConfig) error { info, err := parsePackage(cfg.src) diff --git a/internal/utils/id_parser.go b/internal/utils/id_parser.go index 75aa0c613..c3d8d8d15 100644 --- a/internal/utils/id_parser.go +++ b/internal/utils/id_parser.go @@ -199,11 +199,9 @@ func ResolvePartialID(ctx context.Context, store storage.Storage, input string) return "", fmt.Errorf("no issue found matching %q", input) } - // Sort so the ambiguity error lists IDs deterministically. matches is built - // in SearchIssues return order, whose primary sort is created_at DESC; that - // order is not stable across backends (Postgres timestamp(0) vs Dolt), so an - // unsorted list makes the same ambiguous input print in different orders on - // different storage. Sorting by ID pins the message regardless of backend. + // Sort so the ambiguity error lists IDs deterministically. SearchIssues return + // order is not a contract for ambiguous matches, so sorting by ID pins the same + // message for every storage implementation. sort.Strings(matches) if len(matches) > 1 { diff --git a/open_backend_selection_test.go b/open_backend_selection_test.go new file mode 100644 index 000000000..659f1feea --- /dev/null +++ b/open_backend_selection_test.go @@ -0,0 +1,118 @@ +package beads_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads" +) + +func writeBackendMetadata(t *testing.T, backend string) string { + t.Helper() + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatalf("create .beads: %v", err) + } + metadata := `{"backend":"` + backend + `"}` + if backend == "sqlite" { + // Current SQLite workspaces carry an explicit path marker. A bare + // backend:"sqlite" can be stale metadata from the earlier SQLite era + // and must not be used as a fresh-provisioning fixture (see PR #4740). + metadata = `{"backend":"sqlite","sqlite_path":"beads.db"}` + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(metadata), 0o600); err != nil { + t.Fatalf("write metadata: %v", err) + } + return beadsDir +} + +func TestOpenBestAvailableRoutesSQLite(t *testing.T) { + beadsDir := writeBackendMetadata(t, "sqlite") + store, err := beads.OpenBestAvailable(context.Background(), beadsDir) + if err != nil { + t.Fatalf("OpenBestAvailable(SQLite): %v", err) + } + defer store.Close() + + if err := store.SetConfig(context.Background(), "routing-test", "sqlite"); err != nil { + t.Fatalf("write through SQLite store: %v", err) + } + if got, err := store.GetConfig(context.Background(), "routing-test"); err != nil || got != "sqlite" { + t.Fatalf("read through SQLite store = %q, %v; want sqlite, nil", got, err) + } + if _, err := os.Stat(filepath.Join(beadsDir, "beads.db")); err != nil { + t.Fatalf("SQLite database was not created: %v", err) + } + if _, err := os.Stat(filepath.Join(beadsDir, "embeddeddolt")); !os.IsNotExist(err) { + t.Fatalf("SQLite routing created embedded Dolt storage (stat error: %v)", err) + } +} + +func TestOpenBestAvailableRejectsRemovedBackends(t *testing.T) { + for _, backend := range []string{"postgres", "mysql"} { + t.Run(backend, func(t *testing.T) { + beadsDir := writeBackendMetadata(t, backend) + store, err := beads.OpenBestAvailable(context.Background(), beadsDir) + if store != nil { + _ = store.Close() + t.Fatalf("removed backend %q returned a store", backend) + } + if err == nil || !strings.Contains(err.Error(), "no longer supported") { + t.Fatalf("removed backend error = %v, want rollback explanation", err) + } + if !strings.Contains(err.Error(), "simple") || !strings.Contains(err.Error(), "resource") || !strings.Contains(err.Error(), "export") { + t.Fatalf("removed backend error lacks rationale or migration guidance: %v", err) + } + for _, name := range []string{"embeddeddolt", "dolt", "beads.db"} { + if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { + t.Fatalf("removed backend created %s (stat error: %v)", name, statErr) + } + } + }) + } +} + +func TestOpenBestAvailableRejectsUnknownBackend(t *testing.T) { + beadsDir := writeBackendMetadata(t, "mystery") + store, err := beads.OpenBestAvailable(context.Background(), beadsDir) + if store != nil { + _ = store.Close() + t.Fatal("unknown backend returned a store") + } + if err == nil || !strings.Contains(err.Error(), "not recognized") { + t.Fatalf("unknown backend error = %v, want fail-closed metadata guidance", err) + } + if !strings.Contains(err.Error(), "no storage database was opened or modified") { + t.Fatalf("unknown backend error lacks data-safety guarantee: %v", err) + } + for _, name := range []string{"embeddeddolt", "dolt", "beads.db"} { + if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { + t.Fatalf("unknown backend created %s (stat error: %v)", name, statErr) + } + } +} + +func TestOpenBestAvailableRejectsCorruptMetadata(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatalf("create .beads: %v", err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte("{"), 0o600); err != nil { + t.Fatalf("write metadata: %v", err) + } + + store, err := beads.OpenBestAvailable(context.Background(), beadsDir) + if store != nil { + _ = store.Close() + t.Fatal("corrupt metadata unexpectedly returned a store") + } + if err == nil || !strings.Contains(err.Error(), "metadata") { + t.Fatalf("corrupt metadata error = %v, want metadata load failure", err) + } + if _, statErr := os.Stat(filepath.Join(beadsDir, "embeddeddolt")); !os.IsNotExist(statErr) { + t.Fatalf("corrupt metadata created embedded Dolt storage (stat error: %v)", statErr) + } +} diff --git a/scripts/conformance.sh b/scripts/conformance.sh index 5937e6450..a30623245 100755 --- a/scripts/conformance.sh +++ b/scripts/conformance.sh @@ -4,17 +4,8 @@ # # ./scripts/conformance.sh # -# Two tiers, both reading the backend registry (test/conformance/profiles.go for the -# E2E tier; the per-backend conformance_test.go files for the in-process tier). Add a -# backend = add a profile + a factory; both tiers pick it up. -# -# Backends that need an external service auto-skip when their env is unset: -# BEADS_PG_TEST_URL postgres://user:pass@host:port/db (enables the postgres backend) -# BEADS_PG_PASSWORD optional, if the password is not in the URL -# -# Optional deep gate (bts-rs 523-scenario differential oracle; needs the bts-rs -# checkout + ~50 min, so it is off by default and not part of the per-PR loop): -# CONFORMANCE_DEEP=1 BTS_RS_DIR=/path/to/bts-rs ./scripts/conformance.sh +# Two tiers compare the maintained SQLite implementation against embedded Dolt: +# the in-process storage corpus, then the real-binary CLI corpus. # set -euo pipefail cd "$(dirname "$0")/.." @@ -38,37 +29,6 @@ assert_conformance_passed() { rm -f "$log" } -# require_backend NAME -> success if NAME is listed (comma-separated) in -# BEADS_CONFORMANCE_REQUIRE. CI sets BEADS_CONFORMANCE_REQUIRE=postgres,mysql so a -# backend that self-skips because its BEADS_*_TEST_URL is missing becomes a hard -# failure instead of a silent green; unset locally, the backend self-skips as before. -require_backend() { - local want="$1" tok - local IFS=, - for tok in ${BEADS_CONFORMANCE_REQUIRE:-}; do - [ "$(printf '%s' "$tok" | tr -d '[:space:]')" = "$want" ] && return 0 - done - return 1 -} - -# run_sql_conformance NAME PKG RUN URLVAR -# Run one SQL backend's in-process conformance + wedge gates. When NAME is required -# (see require_backend), run verbose and assert the TestConformance parity oracle -# actually ran and passed — mirroring the embedded-Dolt reference guard so a missing -# BEADS_*_TEST_URL can't silently narrow coverage. Otherwise run plain and let the -# suite self-skip. URLVAR names the env var to set, surfaced in the failure hint. -run_sql_conformance() { - local name="$1" pkg="$2" run="$3" urlvar="$4" - if require_backend "$name"; then - local log - log="$(mktemp)" - CGO_ENABLED=1 go test -tags "$TAGS" -v "$pkg" -run "$run" | tee "$log" - assert_conformance_passed "$log" "$name backend (set $urlvar)" - else - CGO_ENABLED=1 go test -tags "$TAGS" "$pkg" -run "$run" - fi -} - echo "==> Tier 1: in-process store conformance + wedge gates" # The embedded-Dolt backend runs the full backend-agnostic suite (conformance.RunAll) as # the reference oracle every SQL backend is compared against. BEADS_TEST_EMBEDDED_DOLT=1 @@ -82,26 +42,6 @@ embedded_dolt_log="$(mktemp)" BEADS_TEST_EMBEDDED_DOLT=1 CGO_ENABLED=1 go test -tags "$TAGS" -v \ ./internal/storage/embeddeddolt/ -run TestConformance | tee "$embedded_dolt_log" assert_conformance_passed "$embedded_dolt_log" "embedded-Dolt reference (need BEADS_TEST_EMBEDDED_DOLT=1)" -# Each SQL backend now passes the FULL in-process store conformance suite (TestConformance -# = conformance.RunAll, the same ~40 behavior subtests the Dolt reference runs) — the only -# methods it does NOT implement are the genuinely-Dolt-only ones (version-control/remote/ -# sync/…), which RunAll does not exercise. Alongside RunAll: live smoke, the -# interface-completeness audit (shell == deferral allowlist — no SILENT unsupported) plus -# its behavioral complement (every allowlisted method returns typed ErrUnsupported), the -# seed-once regression, and the dialect corpus-PREPARE + password-redaction gates. All -# self-skip without BEADS_PG_TEST_URL — or hard-fail when BEADS_CONFORMANCE_REQUIRE -# lists postgres (see run_sql_conformance), so CI can't go green with the wedge absent. -run_sql_conformance postgres ./internal/storage/postgres/ \ - 'TestPGSmoke|TestInterfaceCompleteness|TestUnsupportedContract|TestConformance|TestSeedOnlyOnFirstProvision' \ - BEADS_PG_TEST_URL -CGO_ENABLED=1 go test -tags "$TAGS" ./internal/storage/pgdialect/ -# MySQL wedge gates (self-skip without BEADS_MYSQL_TEST_URL, or hard-fail when -# BEADS_CONFORMANCE_REQUIRE lists mysql); the dialect rewrite test (the is_blocked 1093 -# workaround) always runs. -run_sql_conformance mysql ./internal/storage/mysql/ \ - 'TestInterfaceCompleteness|TestUnsupportedContract|TestConformance|TestSeedOnlyOnFirstProvision' \ - BEADS_MYSQL_TEST_URL -CGO_ENABLED=1 go test -tags "$TAGS" ./internal/storage/mysqldialect/ # SQLite is embedded (pure-Go), always runs. CGO_ENABLED=1 go test -tags "$TAGS" ./internal/storage/sqlite/ \ -run 'TestInterfaceCompleteness|TestUnsupportedContract|TestConformance|TestSeedOnlyOnFirstProvision' @@ -110,9 +50,4 @@ CGO_ENABLED=1 go test -tags "$TAGS" ./internal/storage/sqlitedialect/ echo "==> Tier 2: end-to-end 'bd init' + CLI conformance (differential vs Dolt)" CGO_ENABLED=1 go test -tags "$TAGS e2e" ./test/conformance/ -if [[ "${CONFORMANCE_DEEP:-0}" == "1" ]]; then - echo "==> Deep: bts-rs 523-scenario differential oracle" - ./scripts/run-oracle-p.sh -fi - echo "==> conformance OK" diff --git a/scripts/run-oracle-p.sh b/scripts/run-oracle-p.sh deleted file mode 100755 index f4ba3597f..000000000 --- a/scripts/run-oracle-p.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# -# Deep conformance gate: the bts-rs 523-scenario differential oracle, PG-vs-Dolt. -# Reference = bd in Dolt mode; candidate = the SAME bd via a thin flag-injector that -# runs native `bd init --backend=postgres`. On-demand only (needs the bts-rs checkout, -# a live Postgres, and ~50 min); invoked by scripts/conformance.sh when CONFORMANCE_DEEP=1. -# -# Env: -# BTS_RS_DIR path to the bts-rs checkout (required) -# BEADS_PG_TEST_URL postgres URL incl. password (required) -# BD path to a prebuilt bd binary (optional; built if unset) -# -set -euo pipefail -cd "$(dirname "$0")/.." -: "${BTS_RS_DIR:?set BTS_RS_DIR to the bts-rs checkout}" -: "${BEADS_PG_TEST_URL:?set BEADS_PG_TEST_URL (postgres URL incl. password)}" - -BD="${BD:-}" -if [[ -z "$BD" ]]; then - BD="$(mktemp -d)/bd" - echo "### building bd -> $BD" - CGO_ENABLED=1 go build -tags gms_pure_go -o "$BD" ./cmd/bd -fi - -# Thin flag-injector: native `bd init` does the metadata + schema + seeding; this only -# supplies the backend flags the backend-agnostic harness cannot know, plus a -# deterministic per-workspace schema and a default prefix so IDs match the reference. -WRAP="$(mktemp)" -cat > "$WRAP" </dev/null | psql "$BEADS_PG_TEST_URL" >/dev/null 2>&1 || true - -cd "$BTS_RS_DIR" -echo "### capturing goldens from bd-Dolt" -BTS_CATALOG=1 BTS_REFERENCE_BD="$BD" cargo run --release -q -p bts-conformance --bin capture_golden 2>&1 | tail -4 -echo "### scoring native bd-PG against fresh goldens" -BTS_CATALOG=1 BTS_CANDIDATE="$WRAP" BTS_DATABASE_URL="$BEADS_PG_TEST_URL" \ - cargo run --release -q -p bts-conformance --bin scoreboard 2>&1 | tail -30 -git -C "$BTS_RS_DIR" checkout crates/bts-conformance/testdata/golden >/dev/null 2>&1 || true -echo "### DONE" diff --git a/test/conformance/e2e_test.go b/test/conformance/e2e_test.go index 5df2983df..d9506d0a2 100644 --- a/test/conformance/e2e_test.go +++ b/test/conformance/e2e_test.go @@ -20,11 +20,9 @@ import ( // scenario is a named sequence of bd CLI invocations. IDs are pinned with --id so // they match across backends; only timestamps/paths need normalization. // -// unordered marks a scenario whose JSON-array output has no contractual order: bd -// sorts by priority then created_at, and Postgres's whole-second timestamps tie where -// Dolt's sub-second ones do not, so equal-priority items created in the same second -// can legitimately differ in order. For these, array elements are sorted by id before -// comparison (matching the bts-rs oracle's multiset semantics for list output). +// unordered marks a scenario whose JSON-array output has no contractual order. For +// these, array elements are sorted by id before comparison (matching the bts-rs +// oracle's multiset semantics for list output). type scenario struct { name string steps [][]string @@ -33,7 +31,7 @@ type scenario struct { // corpus is the backend-agnostic CLI surface exercised end-to-end. It is deliberately // compact (the gc lifecycle + config), and grows over time; the bts-rs 523-scenario -// differential oracle (scripts/run-oracle-p.sh) remains the deep gate. +// differential corpus exercises both maintained storage families. var corpus = []scenario{ {name: "create-show", steps: [][]string{ {"create", "First task", "--id", "cf-a1", "-t", "task", "--json"}, @@ -75,8 +73,7 @@ var corpus = []scenario{ {"config", "unset", "custom.foo"}, {"config", "get", "custom.foo"}, }}, - // Deferred surfaces, present to exercise XFail classification against the - // postgres profile's allowlist (the reference passes these). + // Deferred surfaces exercise the typed unsupported-operation contract. {name: "stats", steps: [][]string{ {"create", "s", "--id", "cf-s1", "-t", "task"}, {"stats", "--json"}, @@ -143,12 +140,10 @@ var corpus = []scenario{ // comment UUIDv7 the CLI harness can't normalize, so it is not a CLI-differential.) // Case-variant config key round-trip. `config set` passes the key verbatim to // store.SetConfig (no ToLower) and namespaced keys bypass the recognized-key - // allowlist, so custom.TeamName reaches the store as-is. On the embedded-Dolt - // reference the key column is case-SENSITIVE (verified at the store layer), so + // allowlist, so custom.TeamName reaches the store as-is. On both maintained + // backends the key column is case-sensitive, so // `config get custom.teamname` does NOT see the value set under custom.TeamName, - // while `config get custom.TeamName` does. config set/get/unset all run on SQL - // backends. Divergence suspect: real MySQL's case-insensitive default collation - // would make custom.teamname resolve to the custom.TeamName row. + // while `config get custom.TeamName` does. {name: "config-case-variant-key", steps: [][]string{ {"config", "set", "custom.TeamName", "example-org"}, {"config", "get", "custom.teamname"}, @@ -164,8 +159,8 @@ var corpus = []scenario{ {"create", "childB", "-t", "task", "--parent", "cf-cp1", "--json"}, {"list", "--json"}, }}, - // (bd mol wisp list order is non-contractual for equal-second ties — Dolt/PG vs - // MySQL/SQLite differ on tie order in the nested wisps array, which the harness can't + // (bd mol wisp list order is non-contractual for equal-second ties — Dolt and + // SQLite can differ on tie order in the nested wisps array, which the harness can't // normalize; ListWisps is covered as a set at the store level.) // All cli-differential scenarios below were driven end-to-end against the // embedded-Dolt reference (bd built with -tags gms_pure_go, `bd init -p cf`) and @@ -310,35 +305,14 @@ func TestConformanceE2E(t *testing.T) { ref := Reference() cands := Candidates() if len(cands) == 0 { - t.Skip("no candidate backends available (set BEADS_PG_TEST_URL for the postgres profile)") + t.Fatal("no candidate backends registered") } - // Make coverage visible: sqlite's Available() is always true, so a run with - // BEADS_PG_TEST_URL/BEADS_MYSQL_TEST_URL unset compares only sqlite yet still - // passes green. Log the candidate set, and honor BEADS_CONFORMANCE_REQUIRE - // (comma-separated profile names) so CI hard-fails when an expected backend is - // absent rather than silently narrowing coverage. + // Make coverage visible in test logs. names := make([]string, len(cands)) for i, c := range cands { names[i] = c.Name } t.Logf("E2E differential candidates: %v (reference=%s)", names, ref.Name) - if req := strings.TrimSpace(os.Getenv("BEADS_CONFORMANCE_REQUIRE")); req != "" { - for _, want := range strings.Split(req, ",") { - if want = strings.TrimSpace(want); want == "" { - continue - } - found := false - for _, n := range names { - if n == want { - found = true - break - } - } - if !found { - t.Fatalf("BEADS_CONFORMANCE_REQUIRE lists %q but that candidate is unavailable (set its BEADS_*_TEST_URL)", want) - } - } - } for _, sc := range corpus { sc := sc t.Run(sc.name, func(t *testing.T) { @@ -365,12 +339,6 @@ func TestConformanceE2E(t *testing.T) { func runScenario(t *testing.T, bin string, p BackendProfile, sc scenario) []string { t.Helper() ws := &Workspace{Dir: t.TempDir()} - if p.NewHandle != nil { - ws.Handle = p.NewHandle() - } - if p.Teardown != nil { - t.Cleanup(func() { p.Teardown(ws) }) - } var env []string if p.Env != nil { env = p.Env(ws) @@ -440,13 +408,10 @@ var ( reRoleWarn = regexp.MustCompile(`(?m)^warning: beads\.role not configured \(GH#2950\)\.\n(?:^ (?:Fix|Or): .*\n?)*`) ) -// normalize removes cross-backend and cross-run noise: workspace path, schema handle, -// timestamps, and random tip banners. Pinned IDs need no normalization. +// normalize removes cross-backend and cross-run noise: workspace path, timestamps, +// and random tip banners. Pinned IDs need no normalization. func normalize(s string, ws *Workspace) string { s = strings.ReplaceAll(s, ws.Dir, "") - if ws.Handle != "" { - s = strings.ReplaceAll(s, ws.Handle, "") - } s = reZeroTS.ReplaceAllString(s, "") s = reTimestamp.ReplaceAllString(s, "") s = reTip.ReplaceAllString(s, "") diff --git a/test/conformance/handle.go b/test/conformance/handle.go deleted file mode 100644 index df0b3b685..000000000 --- a/test/conformance/handle.go +++ /dev/null @@ -1,61 +0,0 @@ -package conformance - -import ( - "context" - "database/sql" - "fmt" - "os" - "sync/atomic" - - gomysql "github.com/go-sql-driver/mysql" - "github.com/steveyegge/beads/internal/storage/pgdialect" -) - -var handleSeq int64 - -// freshSchemaName mints a process-unique Postgres schema name. The pid keeps names -// distinct across concurrent runs and after a crashed run; teardown drops them. -func freshSchemaName() string { - return fmt.Sprintf("e2e_%d_%d", os.Getpid(), atomic.AddInt64(&handleSeq, 1)) -} - -// dropPostgresSchema tears down a workspace's schema. Best effort: a failure here -// only leaves a stray schema, which the next run's freshSchemaName avoids colliding -// with. Uses a raw (non-translating) connection since DROP SCHEMA is native DDL. -func dropPostgresSchema(ws *Workspace) { - if ws.Handle == "" { - return - } - url := os.Getenv("BEADS_PG_TEST_URL") - if url == "" { - return - } - raw, err := pgdialect.OpenRaw(url, "public") - if err != nil { - return - } - defer raw.Close() - _, _ = raw.ExecContext(context.Background(), fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, ws.Handle)) -} - -// dropMySQLDatabase tears down a workspace's database. Best effort. -func dropMySQLDatabase(ws *Workspace) { - if ws.Handle == "" { - return - } - url := os.Getenv("BEADS_MYSQL_TEST_URL") - if url == "" { - return - } - cfg, err := gomysql.ParseDSN(url) - if err != nil { - return - } - cfg.DBName = "" - db, err := sql.Open("mysql", cfg.FormatDSN()) - if err != nil { - return - } - defer db.Close() - _, _ = db.ExecContext(context.Background(), "DROP DATABASE IF EXISTS `"+ws.Handle+"`") -} diff --git a/test/conformance/profiles.go b/test/conformance/profiles.go index 626b3296e..6a1735b31 100644 --- a/test/conformance/profiles.go +++ b/test/conformance/profiles.go @@ -1,24 +1,19 @@ // Package conformance is the end-to-end (real `bd` binary) conformance harness. It // complements the in-process store suite (internal/storage/conformance): that one // exercises a store object; this one exercises the whole CLI path — `bd init`, the -// factory, metadata.json, dialect translation over the real binary, and output +// factory, metadata.json, SQLite dialect translation over the real binary, and output // formatting — differentially against the Dolt reference. // // One declarative registry (Profiles) describes how to create and tear down an // isolated workspace for each backend. Adding a backend is one Profiles entry; the // runner and scripts/conformance.sh pick it up automatically. The suite is behind -// the `e2e` build tag because it shells out to a freshly built bd and needs live -// backends for the non-default profiles. +// the `e2e` build tag because it shells out to a freshly built bd. package conformance -import "os" - // Workspace is one isolated place a backend can be `bd init`-ed. Dir is the working -// directory (its .beads/ is created by init); Handle is a backend-specific isolation -// id (a Postgres schema, a server database, …) used for teardown. +// directory; init creates its .beads directory. type Workspace struct { - Dir string - Handle string + Dir string } // BackendProfile declares everything the E2E runner needs to init and isolate a @@ -37,11 +32,6 @@ type BackendProfile struct { InitArgs func(ws *Workspace) []string // Env returns extra environment (KEY=VALUE) for every bd invocation in ws. Env func(ws *Workspace) []string - // NewHandle mints a fresh isolation id (schema/db name) for a workspace. Empty - // for backends isolated purely by directory (embedded Dolt). - NewHandle func() string - // Teardown releases the isolation (drop schema/db). Best effort; may be nil. - Teardown func(ws *Workspace) // XFail lists scenarios this backend is known not to match the reference on, // each with a reason. They are reported as XFAIL (visible, never masked); an // XFAIL that starts matching is flagged so the list can only shrink. @@ -56,43 +46,6 @@ var Profiles = []BackendProfile{ Available: func() bool { return true }, InitArgs: func(*Workspace) []string { return nil }, }, - { - Name: "postgres", - Available: func() bool { return os.Getenv("BEADS_PG_TEST_URL") != "" }, - NewHandle: freshSchemaName, - InitArgs: func(ws *Workspace) []string { - return []string{ - "--backend=postgres", - "--pg-url=" + os.Getenv("BEADS_PG_TEST_URL"), - "--pg-schema=" + ws.Handle, - } - }, - Env: func(*Workspace) []string { - if pw := os.Getenv("BEADS_PG_PASSWORD"); pw != "" { - return []string{"BEADS_PG_PASSWORD=" + pw} - } - return nil - }, - Teardown: dropPostgresSchema, - }, - { - Name: "mysql", - Available: func() bool { return os.Getenv("BEADS_MYSQL_TEST_URL") != "" }, - NewHandle: freshSchemaName, // a valid MySQL database name - InitArgs: func(ws *Workspace) []string { - return []string{ - "--backend=mysql", - "--mysql-url=" + os.Getenv("BEADS_MYSQL_TEST_URL"), - "--mysql-database=" + ws.Handle, - } - }, - Env: func(*Workspace) []string { - // The server DSN (with password) is used for every command; the - // per-workspace database comes from metadata.json / --mysql-database. - return []string{"BEADS_MYSQL_URL=" + os.Getenv("BEADS_MYSQL_TEST_URL")} - }, - Teardown: dropMySQLDatabase, - }, { Name: "sqlite", Available: func() bool { return true }, // embedded (pure-Go modernc); always runs diff --git a/test/conformance/roundtrip_test.go b/test/conformance/roundtrip_test.go index fea1e97a4..aed1c2d53 100644 --- a/test/conformance/roundtrip_test.go +++ b/test/conformance/roundtrip_test.go @@ -12,12 +12,10 @@ import ( "testing" ) -// The portable JSONL backup path — `bd export` -> git -> `bd import` — is the only -// backup mechanism the non-Dolt backends have (Dolt-native CALL DOLT_BACKUP is -// unsupported on them, and auto-export is gated to Dolt because its change token is -// the Dolt commit hash). The manual round-trip is already lossless on every backend -// (verified live against real Postgres/MySQL); this test pins that so a regression in -// export or import cannot slip through CI. +// The portable JSONL backup path — `bd export` -> git -> `bd import` — is SQLite's +// backup mechanism because Dolt-native CALL DOLT_BACKUP and commit hashes do not +// exist there. This test pins the manual round-trip so a regression in export or +// import cannot slip through CI. // // It is a self-consistency (idempotence) check, not a differential one, so it lives // outside the `corpus`: for each backend it seeds a rich fixture, exports, restores @@ -125,18 +123,11 @@ func restoreAndReexport(t *testing.T, bin string, p BackendProfile, exportJSONL return mustRead(t, out) } -// initRoundTripWorkspace mints an isolated workspace for a backend (fresh temp dir plus -// a fresh handle/schema for the server backends) and runs `bd init`. Prefix "rt" matches -// the pinned rt-* ids in the seed. +// initRoundTripWorkspace creates an isolated temporary workspace and runs `bd init`. +// Prefix "rt" matches the pinned rt-* ids in the seed. func initRoundTripWorkspace(t *testing.T, bin string, p BackendProfile) (*Workspace, []string) { t.Helper() ws := &Workspace{Dir: t.TempDir()} - if p.NewHandle != nil { - ws.Handle = p.NewHandle() - } - if p.Teardown != nil { - t.Cleanup(func() { p.Teardown(ws) }) - } var env []string if p.Env != nil { env = p.Env(ws)