From 0cf3b3ca0924d4ef738bac4536ddad0536db756a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 16 Jul 2026 16:35:51 -0700 Subject: [PATCH] Revert "Simplify the supported backends: roll back the direct PostgreSQL and MySQL adapters" (#4857) --- .github/workflows/conformance.yml | 63 +- .golangci.yml | 8 - CHANGELOG.md | 12 - PROPOSAL-pluggable-storage-backends.md | 17 +- README.md | 37 +- beads.go | 16 +- beads_cgo.go | 24 +- beads_nocgo.go | 22 +- cmd/bd/backend_support.go | 45 - cmd/bd/bootstrap.go | 42 +- cmd/bd/bootstrap_backend_guard_test.go | 293 ------- cmd/bd/bootstrap_test.go | 41 +- cmd/bd/doctor.go | 122 +-- cmd/bd/doctor/deep.go | 11 +- cmd/bd/doctor/deep_test.go | 21 - cmd/bd/doctor/dolt_e2e_test.go | 2 +- cmd/bd/doctor/migration_validation.go | 6 +- cmd/bd/doctor/migration_validation_test.go | 31 +- cmd/bd/doctor/perf_dolt.go | 3 +- cmd/bd/doctor/perf_dolt_test.go | 24 - cmd/bd/doctor/server.go | 17 +- cmd/bd/doctor/server_backend_test.go | 30 - cmd/bd/doctor_backend_guard_test.go | 61 -- cmd/bd/doctor_embedded_test.go | 26 - cmd/bd/doctor_validate_test.go | 2 +- cmd/bd/dolt.go | 111 ++- cmd/bd/init.go | 352 ++++++-- cmd/bd/init_embedded_test.go | 13 +- cmd/bd/init_test.go | 40 +- cmd/bd/main.go | 132 ++- cmd/bd/main_test.go | 6 - cmd/bd/store_factory.go | 19 +- cmd/bd/store_factory_nocgo.go | 18 +- .../store_selection_removed_backend_test.go | 403 --------- cmd/bd/telemetry_redact.go | 129 --- cmd/bd/telemetry_scrub_test.go | 264 +----- cmd/bd/version_tracking.go | 4 - cmd/bd/version_tracking_test.go | 2 +- default.nix | 2 +- docs/CLI_REFERENCE.md | 41 +- docs/architecture/dolt.md | 37 +- docs/architecture/index.md | 16 +- docs/architecture/storage-backends.md | 332 ++++---- docs/cli-reference/doctor.md | 33 +- docs/cli-reference/init.md | 8 +- docs/core-concepts/index.md | 19 +- docs/getting-started/quickstart.md | 4 +- docs/multi-agent/federation.md | 3 +- docs/reference/configuration.md | 55 +- docs/reference/faq.md | 74 +- examples/library-usage/README.md | 10 +- examples/library-usage/main.go | 14 +- examples/library-usage/main_test.go | 10 +- go.mod | 4 + go.sum | 7 + internal/beads/beads.go | 35 +- internal/beads/sqlite_database_path_test.go | 85 -- internal/configfile/backend_messages.go | 38 - internal/configfile/configfile.go | 116 ++- internal/configfile/configfile_test.go | 31 +- internal/configfile/mysql_dsn_test.go | 37 + internal/configfile/postgres_dsn_test.go | 36 + internal/storage/backend_marker.go | 7 +- .../audit_config_metadata_slots_repomtime.go | 5 +- .../audit_labels-comments-events.go | 2 +- .../audit_molecule_wisp_batch_iter.go | 5 +- .../conformance/audit_search-counts-stats.go | 10 +- internal/storage/conformance/claim.go | 104 ++- internal/storage/conformance/conformance.go | 11 +- internal/storage/conformance/portable.go | 8 +- internal/storage/dolt/open.go | 25 - internal/storage/dolt/open_test.go | 44 - internal/storage/embeddeddolt/iter_stubs.go | 4 +- internal/storage/errors_unsupported.go | 2 +- internal/storage/issueops/blocked_state.go | 23 + internal/storage/issueops/count.go | 4 +- internal/storage/issueops/create.go | 7 +- internal/storage/issueops/helpers.go | 7 +- internal/storage/issueops/lease.go | 5 +- internal/storage/iter.go | 2 +- internal/storage/mysql/completeness_test.go | 107 +++ internal/storage/mysql/conformance_test.go | 80 ++ internal/storage/mysql/credential.go | 67 ++ internal/storage/mysql/credential_test.go | 288 +++++++ internal/storage/mysql/dialect.go | 23 + internal/storage/mysql/dsn.go | 53 ++ internal/storage/mysql/fromconfig.go | 57 ++ internal/storage/mysql/redact_test.go | 117 +++ internal/storage/mysql/schema.go | 162 ++++ internal/storage/mysql/schema.sql | 337 ++++++++ internal/storage/mysql/seed_test.go | 54 ++ internal/storage/mysql/store.go | 69 ++ internal/storage/mysql/unsupported.go | 17 + internal/storage/mysql/unsupported_gen.go | 236 ++++++ internal/storage/mysqldialect/credential.go | 83 ++ .../mysqldialect/credential_extra_test.go | 81 ++ .../storage/mysqldialect/credential_test.go | 82 ++ internal/storage/mysqldialect/driver.go | 91 ++ internal/storage/mysqldialect/rewrite.go | 43 + internal/storage/mysqldialect/rewrite_test.go | 123 +++ internal/storage/pgdialect/corpus_test.go | 426 ++++++++++ internal/storage/pgdialect/credential.go | 121 +++ .../pgdialect/credential_extra_test.go | 148 ++++ internal/storage/pgdialect/credential_test.go | 96 +++ internal/storage/pgdialect/driver.go | 190 +++++ internal/storage/pgdialect/redact.go | 214 +++++ .../storage/pgdialect/redact_error_test.go | 188 +++++ .../storage/pgdialect/redact_matrix_test.go | 179 ++++ internal/storage/pgdialect/redact_test.go | 58 ++ internal/storage/pgdialect/translate.go | 777 ++++++++++++++++++ internal/storage/pgdialect/translate_test.go | 98 +++ internal/storage/postgres/collation.go | 49 ++ internal/storage/postgres/collation_test.go | 29 + .../storage/postgres/completeness_test.go | 107 +++ internal/storage/postgres/conformance_test.go | 89 ++ internal/storage/postgres/credential.go | 67 ++ internal/storage/postgres/credential_test.go | 260 ++++++ internal/storage/postgres/dialect.go | 22 + internal/storage/postgres/fromconfig.go | 68 ++ internal/storage/postgres/repro_instr_test.go | 46 ++ internal/storage/postgres/schema.go | 675 +++++++++++++++ internal/storage/postgres/schema_test.go | 155 ++++ internal/storage/postgres/seed_test.go | 57 ++ internal/storage/postgres/smoke_test.go | 104 +++ internal/storage/postgres/store.go | 87 ++ internal/storage/postgres/unsupported.go | 24 + internal/storage/postgres/unsupported_gen.go | 236 ++++++ internal/storage/sqlbuild/sort.go | 10 +- internal/storage/sqlite/completeness_test.go | 14 +- internal/storage/sqlite/dsn.go | 15 +- internal/storage/sqlite/schema.go | 2 +- internal/storage/sqlite/unsupported.go | 4 +- internal/storage/sqlitedialect/driver.go | 4 +- internal/storage/sqlitedialect/translate.go | 16 +- internal/storage/sqlkit/queries.go | 4 +- internal/storage/sqlkit/store.go | 6 +- internal/storage/sqlkit/strategy.go | 13 +- internal/storage/unsupportedgen/gen_test.go | 2 +- internal/storage/unsupportedgen/main.go | 6 +- internal/utils/id_parser.go | 8 +- open_backend_selection_test.go | 118 --- scripts/conformance.sh | 69 +- scripts/run-oracle-p.sh | 56 ++ test/conformance/e2e_test.go | 63 +- test/conformance/handle.go | 61 ++ test/conformance/profiles.go | 55 +- test/conformance/roundtrip_test.go | 21 +- 147 files changed, 8374 insertions(+), 2704 deletions(-) delete mode 100644 cmd/bd/backend_support.go delete mode 100644 cmd/bd/bootstrap_backend_guard_test.go delete mode 100644 cmd/bd/doctor/server_backend_test.go delete mode 100644 cmd/bd/doctor_backend_guard_test.go delete mode 100644 cmd/bd/store_selection_removed_backend_test.go delete mode 100644 cmd/bd/telemetry_redact.go delete mode 100644 internal/beads/sqlite_database_path_test.go delete mode 100644 internal/configfile/backend_messages.go create mode 100644 internal/configfile/mysql_dsn_test.go create mode 100644 internal/configfile/postgres_dsn_test.go create mode 100644 internal/storage/mysql/completeness_test.go create mode 100644 internal/storage/mysql/conformance_test.go create mode 100644 internal/storage/mysql/credential.go create mode 100644 internal/storage/mysql/credential_test.go create mode 100644 internal/storage/mysql/dialect.go create mode 100644 internal/storage/mysql/dsn.go create mode 100644 internal/storage/mysql/fromconfig.go create mode 100644 internal/storage/mysql/redact_test.go create mode 100644 internal/storage/mysql/schema.go create mode 100644 internal/storage/mysql/schema.sql create mode 100644 internal/storage/mysql/seed_test.go create mode 100644 internal/storage/mysql/store.go create mode 100644 internal/storage/mysql/unsupported.go create mode 100644 internal/storage/mysql/unsupported_gen.go create mode 100644 internal/storage/mysqldialect/credential.go create mode 100644 internal/storage/mysqldialect/credential_extra_test.go create mode 100644 internal/storage/mysqldialect/credential_test.go create mode 100644 internal/storage/mysqldialect/driver.go create mode 100644 internal/storage/mysqldialect/rewrite.go create mode 100644 internal/storage/mysqldialect/rewrite_test.go create mode 100644 internal/storage/pgdialect/corpus_test.go create mode 100644 internal/storage/pgdialect/credential.go create mode 100644 internal/storage/pgdialect/credential_extra_test.go create mode 100644 internal/storage/pgdialect/credential_test.go create mode 100644 internal/storage/pgdialect/driver.go create mode 100644 internal/storage/pgdialect/redact.go create mode 100644 internal/storage/pgdialect/redact_error_test.go create mode 100644 internal/storage/pgdialect/redact_matrix_test.go create mode 100644 internal/storage/pgdialect/redact_test.go create mode 100644 internal/storage/pgdialect/translate.go create mode 100644 internal/storage/pgdialect/translate_test.go create mode 100644 internal/storage/postgres/collation.go create mode 100644 internal/storage/postgres/collation_test.go create mode 100644 internal/storage/postgres/completeness_test.go create mode 100644 internal/storage/postgres/conformance_test.go create mode 100644 internal/storage/postgres/credential.go create mode 100644 internal/storage/postgres/credential_test.go create mode 100644 internal/storage/postgres/dialect.go create mode 100644 internal/storage/postgres/fromconfig.go create mode 100644 internal/storage/postgres/repro_instr_test.go create mode 100644 internal/storage/postgres/schema.go create mode 100644 internal/storage/postgres/schema_test.go create mode 100644 internal/storage/postgres/seed_test.go create mode 100644 internal/storage/postgres/smoke_test.go create mode 100644 internal/storage/postgres/store.go create mode 100644 internal/storage/postgres/unsupported.go create mode 100644 internal/storage/postgres/unsupported_gen.go delete mode 100644 open_backend_selection_test.go create mode 100755 scripts/run-oracle-p.sh create mode 100644 test/conformance/handle.go diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 9b0f55a73..76cfc9caf 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,7 +1,12 @@ name: Conformance -# Runs the same two-tier Dolt/SQLite storage conformance suite developers run -# locally: the in-process store corpus and the real-binary CLI differential. +# Runs the two-tier storage-backend conformance suite against REAL databases β€” +# the same scripts/conformance.sh developers run locally, so local == CI. Tier 1 +# is the in-process store suite plus the wedge gates; Tier 2 is the end-to-end +# `bd init` + CLI differential (every SQL backend compared to the Dolt reference). +# Postgres and MySQL run as service containers; Dolt (embedded) and SQLite are +# in-process. The deep bts-rs 523-scenario oracle stays on-demand (CONFORMANCE_DEEP), +# not in this per-PR job. on: pull_request: @@ -18,8 +23,40 @@ env: jobs: conformance: - name: Storage backend conformance (Dolt + SQLite) + name: Storage backend conformance (real DBs) runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: bts + POSTGRES_PASSWORD: bts + POSTGRES_DB: bdwedge + # Code-point collation parity with Dolt (utf8mb4_0900_bin). The pg backend + # provisions per-workspace SCHEMAS inside this one database and can't set + # per-column COLLATE "C" (it breaks the shared recursive CTEs; see + # internal/storage/postgres/schema.go), so byte-order parity has to be + # guaranteed at DB-creation time via a C-locale cluster. + POSTGRES_INITDB_ARGS: "--lc-collate=C --lc-ctype=C" + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U bts -d bdwedge" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + mysql: + image: mysql:8 + env: + MYSQL_ROOT_PASSWORD: bts + MYSQL_DATABASE: bdwedge + ports: + - 3306:3306 + options: >- + --health-cmd "mysqladmin ping -uroot -pbts --silent" + --health-interval 5s + --health-timeout 5s + --health-retries 30 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v6 @@ -28,5 +65,23 @@ jobs: with: go-version-file: 'go.mod' - - name: Run conformance (Dolt + SQLite) + # The MySQL image's root only auths from localhost inside the container; the + # backend connects over TCP as a distinct host, so create a TCP-reachable user + # matching BEADS_MYSQL_TEST_URL. + - name: Provision MySQL user + run: | + mysql -h 127.0.0.1 -P 3306 -uroot -pbts -e " + CREATE USER IF NOT EXISTS 'bts'@'%' IDENTIFIED BY 'bts'; + GRANT ALL PRIVILEGES ON *.* TO 'bts'@'%' WITH GRANT OPTION; + FLUSH PRIVILEGES;" + + - name: Run conformance (Dolt + Postgres + MySQL + SQLite) + env: + BEADS_PG_TEST_URL: "postgres://bts:bts@127.0.0.1:5432/bdwedge" + BEADS_PG_PASSWORD: "bts" + BEADS_MYSQL_TEST_URL: "bts:bts@tcp(127.0.0.1:3306)/" + # Hard-fail (Tier 1 run_sql_conformance + Tier 2 e2e) if either real backend + # is absent, so removing/renaming a *_TEST_URL can't silently drop coverage + # while the gate still reports green. + BEADS_CONFORMANCE_REQUIRE: "postgres,mysql" run: ./scripts/conformance.sh diff --git a/.golangci.yml b/.golangci.yml index 9dfdfad6a..9982d481c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -158,14 +158,6 @@ linters: linters: - gosec text: "G705" - # G705: XSS via taint in CLI progress/warning/error output β€” every flagged - # sink is fmt.Fprint* to the terminal (os.Stderr or cobra's stdout/stderr - # writers), never a browser context (surfaced by the gosec bundled with - # golangci-lint v2.10) - - path: 'cmd/bd/(ado|doctor|errors|github|gitlab|hooks|jira|linear|main|notion|sync_push_pull)\.go' - linters: - - gosec - text: "G705" # errcheck: fmt.Fprintf errors are noise in CLI help text generation - path: 'cmd/bd/help_all\.go' linters: diff --git a/CHANGELOG.md b/CHANGELOG.md index f6391c4e4..b94004485 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,18 +9,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- **Storage backend scope simplified** (bd-sadcd). The recently merged direct - PostgreSQL and MySQL adapters have been rolled back before entering a tagged - release. Supporting additional general-purpose server databases introduced - dialect, credential, schema-lifecycle, migration, CI, and operational - complexity at odds with our goal of keeping Beads as simple as possible and - consuming as few resources as possible. The storage interface, shared issue - core, SQLite implementation, and conformance harness remain; embedded Dolt, - Dolt server, and SQLite are the supported storage paths. Dolt server mode and - its use of the MySQL wire protocol are unchanged. Existing PostgreSQL or - MySQL workspaces stop before their configured databases are opened or - modified; see the [migration - guide](docs/architecture/storage-backends.md#existing-postgresql-or-mysql-workspaces). - **Cross-type blocking dependencies are now allowed** (bd-wg7ve, [#4034](https://github.com/gastownhall/beads/pull/4034)). `bd dep add ` β€” gating a work item on an epic (program) diff --git a/PROPOSAL-pluggable-storage-backends.md b/PROPOSAL-pluggable-storage-backends.md index 0fc04a82d..16e2edda3 100644 --- a/PROPOSAL-pluggable-storage-backends.md +++ b/PROPOSAL-pluggable-storage-backends.md @@ -1,6 +1,6 @@ # PROPOSAL: Pluggable Storage Backends -**Status:** Historical β€” partially implemented, then superseded on 2026-07-16 +**Status:** Draft for review (survived one adversarial review round; defects folded in) **Date:** 2026-07-02 **Informed by:** the bts-rs Rust spike (a from-scratch Rust reimplementation of the gc-contract surface that validated a backend-agnostic storage seam at 299/299 byte-parity @@ -8,21 +8,6 @@ across two radically different backends), a five-dimension recon of this repo's architecture, and a two-reviewer adversarial verification pass. All file:line citations verified against `worktree-beads-new`. -> **Superseded scope:** This proposal is retained as the design record for the -> storage interface, shared issue core, SQLite adapter, and differential -> conformance work that grew from it. Those pieces remain. The direct -> PostgreSQL and MySQL adapters were rolled back before entering a tagged -> release because supporting additional general-purpose server databases adds -> substantial dialect, credential, schema-lifecycle, migration, CI, and -> operational complexity. Our goal is to keep Beads as simple as possible and -> consume as few resources as possible. The supported paths are now embedded -> Dolt, Dolt server, and SQLite; Dolt server's use of the MySQL wire protocol is -> unchanged. See [Storage Backends](docs/architecture/storage-backends.md) for -> the current design. - -Everything below this notice is retained as historical design material, not as -current backend support or migration guidance. - --- ## 1. End state diff --git a/README.md b/README.md index dbf5d8e42..9fc9e4c0a 100644 --- a/README.md +++ b/README.md @@ -126,26 +126,24 @@ See [docs/reference/antivirus.md](docs/reference/antivirus.md) for Windows AV fa ## πŸ’Ύ Storage Modes -Beads keeps storage behind one interface and supports three deployment paths: +Beads uses [Dolt](https://github.com/dolthub/dolt) as its database. Two modes: -- **Embedded Dolt (default)** β€” `bd init`. Dolt runs in-process, data lives in - `.beads/embeddeddolt/`, and writes are file-locked. Recommended for most users. -- **Dolt server** β€” `bd init --server`. Connects to a `dolt sql-server` for - multiple concurrent writers. Managed repo-local mode uses `.beads/dolt/`; - shared and external servers manage their own data location. -- **SQLite** β€” `bd init --backend=sqlite`. Stores data in a small local database - file with no server or credentials. It does not provide Dolt history or sync. +- **Embedded (default)** β€” `bd init`. Dolt runs in-process, data lives in + `.beads/embeddeddolt/`, single writer. Recommended for most users. +- **Server** β€” `bd init --server`. Connects to an external `dolt sql-server` + for multiple concurrent writers; data lives in `.beads/dolt/`. -The Dolt paths support cross-machine sync with `bd dolt push` / `bd dolt pull` -against `refs/dolt/data` on your git remote. `.beads/issues.jsonl` is an export -for viewers and interchange, not the source of truth or a full backup. Back up -and migrate between Dolt modes with `bd backup`; reclaim Dolt space with -`bd prune` / `bd purge`. +Cross-machine sync uses `bd dolt push` / `bd dolt pull` against +`refs/dolt/data` on your git remote; `.beads/issues.jsonl` is an export for +viewers and interchange, not the source of truth or a backup. Back up and +migrate between modes with `bd backup`; reclaim space with `bd prune` / +`bd purge`. -See the [Dolt backend guide](docs/architecture/dolt.md) for connection flags, -sockets, maintenance, backup, and migration. See -[Storage Backends](docs/architecture/storage-backends.md) for the supported -Dolt and SQLite paths and their trade-offs. +Full detail β€” connection flags, sockets, maintenance, backup, and migration β€” +in the [Dolt backend guide](docs/architecture/dolt.md). Prefer a different +database? [Storage Backends](docs/architecture/storage-backends.md) covers +Postgres, MySQL, and SQLite (Dolt stays the default and the only backend +with history). ## 🌐 Community Tools @@ -155,9 +153,8 @@ See [docs/related-projects.md](docs/related-projects.md) for adjacent or complem ## πŸš€ Git-Free Usage -Storage choice and Git integration are independent. Embedded Dolt, Dolt -server, and SQLite can all be used without Git; hooks, repo discovery, and -identity integration are optional. +Beads works without git. The Dolt database is the storage backend β€” git +integration (hooks, repo discovery, identity) is optional. ```bash # Initialize without git diff --git a/beads.go b/beads.go index 12f055a52..2976ce3d3 100644 --- a/beads.go +++ b/beads.go @@ -11,7 +11,6 @@ import ( "context" "github.com/steveyegge/beads/internal/beads" - "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/types" @@ -20,15 +19,6 @@ import ( // Storage is the interface for beads storage operations type Storage = beads.Storage -func configuredBackendUnavailable(backend string) error { - switch backend { - case configfile.BackendPostgres, configfile.BackendMySQL: - return configfile.RemovedBackendError(backend) - default: - return configfile.UnknownBackendError(backend) - } -} - // Transaction provides atomic multi-operation support within a database transaction. // Use Storage.RunInTransaction() to obtain a Transaction instance. type Transaction = beads.Transaction @@ -74,9 +64,9 @@ func Open(ctx context.Context, dbPath string) (Storage, error) { return dolt.New(ctx, &dolt.Config{Path: dbPath, CreateIfMissing: true}) } -// OpenFromConfig opens the Dolt implementation using configuration from -// metadata.json. Unlike Open, this respects Dolt server mode settings and database -// name configuration. Use OpenBestAvailable when metadata may select SQLite. +// OpenFromConfig opens a beads database using configuration from metadata.json. +// Unlike Open, this respects Dolt server mode settings and database name +// configuration, connecting to the Dolt SQL server when dolt_mode is "server". // beadsDir is the path to the .beads directory. func OpenFromConfig(ctx context.Context, beadsDir string) (Storage, error) { return dolt.NewFromConfigWithOptions(ctx, beadsDir, &dolt.Config{CreateIfMissing: true}) diff --git a/beads_cgo.go b/beads_cgo.go index 4bcc3213d..a5736e73c 100644 --- a/beads_cgo.go +++ b/beads_cgo.go @@ -4,43 +4,25 @@ package beads import ( "context" - "fmt" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/storage/embeddeddolt" - beadssqlite "github.com/steveyegge/beads/internal/storage/sqlite" ) // OpenBestAvailable opens a beads database using the best available backend // for the given .beads directory. It reads metadata.json to determine the // configured mode: // -// - Embedded Dolt (default): Opens via the CGo embedded Dolt engine. -// - Dolt server: Connects to a dolt sql-server via OpenFromConfig. -// - SQLite: Opens the configured local SQLite database. +// - Embedded mode (default): Opens via the CGo embedded Dolt engine. +// - Server mode: Connects to an external dolt sql-server via OpenFromConfig. // // The returned Storage must be closed when no longer needed. // // beadsDir is the path to the .beads directory. func OpenBestAvailable(ctx context.Context, beadsDir string) (Storage, error) { cfg, err := configfile.Load(beadsDir) - if err != nil { - return nil, fmt.Errorf("loading storage metadata: %w", err) - } - if cfg == nil { - cfg = configfile.DefaultConfig() - } - if !configfile.IsSupportedBackend(cfg.Backend) { - return nil, configuredBackendUnavailable(cfg.Backend) - } - - switch cfg.GetBackend() { - case configfile.BackendSQLite: - return beadssqlite.NewFromConfig(ctx, beadsDir) - } - - if cfg.IsDoltServerMode() { + if err == nil && cfg != nil && cfg.IsDoltServerMode() { store, err := dolt.NewFromConfig(ctx, beadsDir) if err != nil { return nil, err diff --git a/beads_nocgo.go b/beads_nocgo.go index d2dcebc16..58ebdab77 100644 --- a/beads_nocgo.go +++ b/beads_nocgo.go @@ -8,32 +8,16 @@ import ( "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/dolt" - beadssqlite "github.com/steveyegge/beads/internal/storage/sqlite" ) // OpenBestAvailable opens a beads database using the best available backend -// for the given .beads directory. In non-CGO builds, Dolt server and SQLite are -// supported; embedded Dolt returns an error directing the user to server mode. +// for the given .beads directory. In non-CGO builds, only server mode is +// supported; embedded mode returns an error directing the user to server mode. // // beadsDir is the path to the .beads directory. func OpenBestAvailable(ctx context.Context, beadsDir string) (Storage, error) { cfg, err := configfile.Load(beadsDir) - if err != nil { - return nil, fmt.Errorf("loading storage metadata: %w", err) - } - if cfg == nil { - cfg = configfile.DefaultConfig() - } - if !configfile.IsSupportedBackend(cfg.Backend) { - return nil, configuredBackendUnavailable(cfg.Backend) - } - - switch cfg.GetBackend() { - case configfile.BackendSQLite: - return beadssqlite.NewFromConfig(ctx, beadsDir) - } - - if cfg.IsDoltServerMode() { + if err == nil && cfg != nil && cfg.IsDoltServerMode() { store, err := dolt.NewFromConfig(ctx, beadsDir) if err != nil { return nil, err diff --git a/cmd/bd/backend_support.go b/cmd/bd/backend_support.go deleted file mode 100644 index 3da2a2de6..000000000 --- a/cmd/bd/backend_support.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/steveyegge/beads/internal/configfile" -) - -func validateConfiguredBackend(cfg *configfile.Config) error { - if cfg == nil { - return nil - } - switch cfg.Backend { - case configfile.BackendPostgres, configfile.BackendMySQL: - return configfile.RemovedBackendError(cfg.Backend) - case "", configfile.BackendDolt, configfile.BackendSQLite: - return nil - default: - return configfile.UnknownBackendError(cfg.Backend) - } -} - -func requireDoltBackend(cfg *configfile.Config) error { - if err := validateConfiguredBackend(cfg); err != nil { - return err - } - if cfg != nil && cfg.GetBackend() != configfile.BackendDolt { - return fmt.Errorf("not using Dolt backend (configured backend %q)", cfg.GetBackend()) - } - return nil -} - -func loadDoltBackendConfig(beadsDir string) (*configfile.Config, error) { - cfg, err := configfile.Load(beadsDir) - if err != nil { - return nil, fmt.Errorf("loading config: %w", err) - } - if cfg == nil { - cfg = configfile.DefaultConfig() - } - if err := requireDoltBackend(cfg); err != nil { - return nil, err - } - return cfg, nil -} diff --git a/cmd/bd/bootstrap.go b/cmd/bd/bootstrap.go index 5e8521ed5..3d0db9667 100644 --- a/cmd/bd/bootstrap.go +++ b/cmd/bd/bootstrap.go @@ -191,21 +191,12 @@ Examples: // name (e.g. dolt_database). Without this, server-mode rigs get the // default name "beads" instead of their configured name. (GH#3029) cfg, err := configfile.Load(beadsDir) - if err != nil { - return HandleError("failed to load %s: %v; no storage database was opened or modified; fix or restore metadata.json and retry", configfile.ConfigPath(beadsDir), err) - } - if cfg == nil { - cfg, err = findParentConfig(beadsDir) - if err != nil { - return HandleError("failed to load ancestor storage metadata: %v; no storage database was opened or modified; fix or restore metadata.json and retry", err) - } + if err != nil || cfg == nil { + cfg = findParentConfig(beadsDir) } if cfg == nil { cfg = configfile.DefaultConfig() } - if err := requireBootstrapDoltBackend(cfg); err != nil { - return HandleError("%v", err) - } resolvedCfg, repairMsg, err := applyBootstrapMetadataRepair(beadsDir, cfg, !dryRun) if err != nil { @@ -279,16 +270,6 @@ func noWorkspaceBootstrapPayload() map[string]interface{} { } } -func requireBootstrapDoltBackend(cfg *configfile.Config) error { - if err := validateConfiguredBackend(cfg); err != nil { - return err - } - if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { - return fmt.Errorf("bd bootstrap is not supported for SQLite workspaces; the configured SQLite database was not modified; run bd commands normally for this workspace") - } - return nil -} - func detectBootstrapAction(beadsDir string, cfg *configfile.Config) BootstrapPlan { plan := BootstrapPlan{ BeadsDir: beadsDir, @@ -532,9 +513,6 @@ func confirmPrompt(message string, nonInteractive bool) bool { } func executeBootstrapPlan(plan BootstrapPlan, cfg *configfile.Config, nonInteractive bool) error { - if err := requireBootstrapDoltBackend(cfg); err != nil { - return err - } if !confirmPrompt("Proceed?", nonInteractive) { fmt.Fprintf(os.Stderr, "Aborted.\n") return nil @@ -977,10 +955,8 @@ func isNonInteractiveBootstrap(flagValue bool) bool { // findParentConfig walks up from beadsDir's parent looking for a // .beads/metadata.json in ancestor directories. This handles the case where a // rig subdirectory (its own git repo) doesn't have a local .beads but its -// parent workspace does. Returns nil if no parent config is found. A malformed -// or unreadable ancestor metadata file is authoritative and returned as an error; -// bootstrap must not skip it and select a more distant workspace or defaults. -func findParentConfig(beadsDir string) (*configfile.Config, error) { +// parent workspace does. Returns nil if no parent config is found. +func findParentConfig(beadsDir string) *configfile.Config { // Start from the parent of beadsDir's enclosing directory. // beadsDir is typically "/.beads", so we start from 's parent. start := filepath.Dir(filepath.Dir(beadsDir)) @@ -988,12 +964,8 @@ func findParentConfig(beadsDir string) (*configfile.Config, error) { for dir := start; dir != "/" && dir != "."; { candidate := filepath.Join(dir, ".beads") - cfg, err := configfile.Load(candidate) - if err != nil { - return nil, fmt.Errorf("%s: %w", configfile.ConfigPath(candidate), err) - } - if cfg != nil { - return cfg, nil + if cfg, err := configfile.Load(candidate); err == nil && cfg != nil { + return cfg } // Don't search above $HOME @@ -1007,7 +979,7 @@ func findParentConfig(beadsDir string) (*configfile.Config, error) { } dir = parent } - return nil, nil + return nil } func init() { diff --git a/cmd/bd/bootstrap_backend_guard_test.go b/cmd/bd/bootstrap_backend_guard_test.go deleted file mode 100644 index 9988f4560..000000000 --- a/cmd/bd/bootstrap_backend_guard_test.go +++ /dev/null @@ -1,293 +0,0 @@ -package main - -import ( - "bytes" - "crypto/sha256" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - "github.com/steveyegge/beads/internal/configfile" -) - -func TestBootstrapRejectsRemovedBackendsBeforeWorkspaceWrites(t *testing.T) { - bd := buildBDForInitTests(t) - - for _, backend := range []string{configfile.BackendPostgres, configfile.BackendMySQL} { - backend := backend - for _, args := range [][]string{{"bootstrap", "--dry-run"}, {"bootstrap", "--yes"}} { - args := args - mode := "execute" - if len(args) > 1 && args[1] == "--dry-run" { - mode = "dry-run" - } - - t.Run(backend+"/"+mode, func(t *testing.T) { - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(beadsDir, 0o700); err != nil { - t.Fatalf("create .beads: %v", err) - } - - metadata := []byte(fmt.Sprintf("{\n \"database\": \"legacy.db\",\n \"backend\": %q,\n \"project_id\": \"legacy-project\"\n}\n", backend)) - metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) - if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { - t.Fatalf("write metadata.json: %v", err) - } - - cmd := exec.Command(bd, args...) - cmd.Dir = root - cmd.Env = bootstrapBackendGuardEnv(root, beadsDir) - out, err := cmd.CombinedOutput() - if err == nil { - t.Errorf("bd %s unexpectedly succeeded for removed backend %q:\n%s", strings.Join(args, " "), backend, out) - } - - message := strings.ToLower(string(out)) - for _, want := range []string{ - "no longer supported", - "general-purpose server databases", - "simple and resource-light", - "was not opened", - "export", - "dolt", - "sqlite", - } { - if !strings.Contains(message, want) { - t.Errorf("bd %s error for %q missing %q:\n%s", strings.Join(args, " "), backend, want, message) - } - } - - after, readErr := os.ReadFile(metadataPath) - if readErr != nil { - t.Fatalf("read metadata.json after rejected bootstrap: %v", readErr) - } - if !bytes.Equal(after, metadata) { - t.Errorf("bd %s rewrote metadata.json for removed backend %q:\nbefore:\n%s\nafter:\n%s", strings.Join(args, " "), backend, metadata, after) - } - assertNoBootstrapStorageArtifacts(t, beadsDir) - }) - } - } -} - -func TestBootstrapRejectsUnknownBackendBeforeWorkspaceWrites(t *testing.T) { - bd := buildBDForInitTests(t) - - for _, args := range [][]string{{"bootstrap", "--dry-run"}, {"bootstrap", "--yes"}} { - mode := "execute" - if args[1] == "--dry-run" { - mode = "dry-run" - } - - t.Run(mode, func(t *testing.T) { - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(beadsDir, 0o700); err != nil { - t.Fatalf("create .beads: %v", err) - } - - metadata := []byte("{\n \"backend\": \"mystery\",\n \"project_id\": \"legacy-project\"\n}\n") - metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) - if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { - t.Fatalf("write metadata.json: %v", err) - } - - cmd := exec.Command(bd, args...) - cmd.Dir = root - cmd.Env = bootstrapBackendGuardEnv(root, beadsDir) - out, err := cmd.CombinedOutput() - if err == nil { - t.Errorf("bd %s unexpectedly accepted an unknown backend:\n%s", strings.Join(args, " "), out) - } - message := string(out) - for _, want := range []string{"not recognized", "no storage database was opened or modified", "dolt", "sqlite"} { - if !strings.Contains(message, want) { - t.Errorf("bd %s error missing %q:\n%s", strings.Join(args, " "), want, message) - } - } - - after, readErr := os.ReadFile(metadataPath) - if readErr != nil { - t.Fatalf("read metadata.json after rejected bootstrap: %v", readErr) - } - if !bytes.Equal(after, metadata) { - t.Errorf("bd %s rewrote unknown-backend metadata:\nbefore:\n%s\nafter:\n%s", strings.Join(args, " "), metadata, after) - } - assertNoBootstrapStorageArtifacts(t, beadsDir) - }) - } -} - -func TestBootstrapRejectsCorruptMetadataBeforeWorkspaceWrites(t *testing.T) { - bd := buildBDForInitTests(t) - - for _, args := range [][]string{{"bootstrap", "--dry-run"}, {"bootstrap", "--yes"}} { - mode := "execute" - if args[1] == "--dry-run" { - mode = "dry-run" - } - - t.Run(mode, func(t *testing.T) { - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(beadsDir, 0o700); err != nil { - t.Fatalf("create .beads: %v", err) - } - - metadata := []byte("{\n") - metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) - if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { - t.Fatalf("write corrupt metadata.json: %v", err) - } - - cmd := exec.Command(bd, args...) - cmd.Dir = root - cmd.Env = bootstrapBackendGuardEnv(root, beadsDir) - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("bd %s unexpectedly ignored corrupt metadata:\n%s", strings.Join(args, " "), out) - } - message := strings.ToLower(string(out)) - for _, want := range []string{"metadata.json", "no storage database was opened or modified"} { - if !strings.Contains(message, want) { - t.Errorf("bd %s error missing %q:\n%s", strings.Join(args, " "), want, out) - } - } - - after, readErr := os.ReadFile(metadataPath) - if readErr != nil { - t.Fatalf("read corrupt metadata after rejected bootstrap: %v", readErr) - } - if !bytes.Equal(after, metadata) { - t.Errorf("bd %s rewrote corrupt metadata:\nbefore: %q\nafter: %q", strings.Join(args, " "), metadata, after) - } - assertNoBootstrapStorageArtifacts(t, beadsDir) - }) - } -} - -func TestBootstrapDoesNotConvertExistingSQLiteWorkspace(t *testing.T) { - bd := buildBDForInitTests(t) - - for _, args := range [][]string{{"bootstrap", "--dry-run"}, {"bootstrap", "--yes"}} { - args := args - mode := "execute" - if len(args) > 1 && args[1] == "--dry-run" { - mode = "dry-run" - } - - t.Run(mode, func(t *testing.T) { - root := t.TempDir() - for _, gitArgs := range [][]string{ - {"init", "-q"}, - {"config", "user.email", "test@test.com"}, - {"config", "user.name", "Test"}, - {"config", "core.hooksPath", ".git/hooks"}, - } { - runGitForBootstrapTest(t, root, gitArgs...) - } - cmd := exec.Command(bd, "init", "--quiet", "--backend=sqlite", "--prefix=sqliteguard") - cmd.Dir = root - cmd.Env = bootstrapBackendGuardEnv(root, filepath.Join(root, ".beads")) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("initialize SQLite workspace: %v\n%s", err, out) - } - - beadsDir := filepath.Join(root, ".beads") - metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) - databasePath := filepath.Join(beadsDir, "beads.db") - metadataBefore := readBootstrapGuardFile(t, metadataPath) - databaseBefore := sha256.Sum256(readBootstrapGuardFile(t, databasePath)) - localVersionBefore, hadLocalVersion := readOptionalBootstrapGuardFile(t, filepath.Join(beadsDir, ".local_version")) - - cmd = exec.Command(bd, args...) - cmd.Dir = root - cmd.Env = bootstrapBackendGuardEnv(root, beadsDir) - out, err := cmd.CombinedOutput() - if err == nil { - t.Errorf("bd %s unexpectedly accepted an existing SQLite workspace:\n%s", strings.Join(args, " "), out) - } - - message := strings.ToLower(string(out)) - if !strings.Contains(message, "bootstrap") || !strings.Contains(message, "sqlite") || - !(strings.Contains(message, "not support") || strings.Contains(message, "unsupported") || - strings.Contains(message, "cannot") || strings.Contains(message, "refus")) { - t.Errorf("bd %s should fail with explicit SQLite bootstrap guidance:\n%s", strings.Join(args, " "), out) - } - - metadataAfter := readBootstrapGuardFile(t, metadataPath) - if !bytes.Equal(metadataAfter, metadataBefore) { - t.Errorf("bd %s rewrote SQLite metadata.json:\nbefore:\n%s\nafter:\n%s", strings.Join(args, " "), metadataBefore, metadataAfter) - } - databaseAfter := sha256.Sum256(readBootstrapGuardFile(t, databasePath)) - if databaseAfter != databaseBefore { - t.Errorf("bd %s modified the existing SQLite database", strings.Join(args, " ")) - } - localVersionAfter, hasLocalVersion := readOptionalBootstrapGuardFile(t, filepath.Join(beadsDir, ".local_version")) - if hasLocalVersion != hadLocalVersion || !bytes.Equal(localVersionAfter, localVersionBefore) { - t.Errorf("bd %s created or modified SQLite workspace version tracking", strings.Join(args, " ")) - } - for _, name := range []string{"embeddeddolt", "dolt"} { - path := filepath.Join(beadsDir, name) - if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { - t.Errorf("bd %s converted SQLite by creating Dolt state at %s (stat error: %v)", strings.Join(args, " "), path, statErr) - } - } - }) - } -} - -func bootstrapBackendGuardEnv(home, beadsDir string) []string { - env := make([]string, 0, len(os.Environ())+7) - for _, entry := range os.Environ() { - if strings.HasPrefix(entry, "BEADS_") || strings.HasPrefix(entry, "BD_") || strings.HasPrefix(entry, "HOME=") { - continue - } - env = append(env, entry) - } - return append(env, - "HOME="+home, - "BEADS_DIR="+beadsDir, - "BEADS_DOLT_AUTO_START=0", - "BEADS_NO_DAEMON=1", - "BEADS_TEST_IGNORE_REPO_CONFIG=1", - "BD_DISABLE_METRICS=1", - "BD_DISABLE_EVENT_FLUSH=1", - "BD_NON_INTERACTIVE=1", - ) -} - -func assertNoBootstrapStorageArtifacts(t *testing.T, beadsDir string) { - t.Helper() - for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version"} { - path := filepath.Join(beadsDir, name) - if _, err := os.Stat(path); !os.IsNotExist(err) { - t.Errorf("rejected bootstrap created local state at %s (stat error: %v)", path, err) - } - } -} - -func readBootstrapGuardFile(t *testing.T, path string) []byte { - t.Helper() - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read %s: %v", path, err) - } - return data -} - -func readOptionalBootstrapGuardFile(t *testing.T, path string) ([]byte, bool) { - t.Helper() - data, err := os.ReadFile(path) - if os.IsNotExist(err) { - return nil, false - } - if err != nil { - t.Fatalf("read %s: %v", path, err) - } - return data, true -} diff --git a/cmd/bd/bootstrap_test.go b/cmd/bd/bootstrap_test.go index f274caed2..92be0d074 100644 --- a/cmd/bd/bootstrap_test.go +++ b/cmd/bd/bootstrap_test.go @@ -919,10 +919,7 @@ func TestBootstrapRigSubdirUsesParentDBName(t *testing.T) { cfg, cfgErr := configfile.Load(synthesizedDir) if cfgErr != nil || cfg == nil { // This is the fix path: search parent directories for metadata.json - cfg, cfgErr = findParentConfig(synthesizedDir) - if cfgErr != nil { - t.Fatalf("findParentConfig: %v", cfgErr) - } + cfg = findParentConfig(synthesizedDir) } if cfg == nil { cfg = configfile.DefaultConfig() @@ -942,37 +939,6 @@ func TestBootstrapRigSubdirUsesParentDBName(t *testing.T) { } } -func TestFindParentConfigDoesNotSkipCorruptNearestAncestor(t *testing.T) { - root := t.TempDir() - higherBeadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(higherBeadsDir, 0o750); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(higherBeadsDir, "metadata.json"), []byte(`{"dolt_database":"wrong_higher_database"}`), 0o600); err != nil { - t.Fatal(err) - } - - nearestBeadsDir := filepath.Join(root, "workspace", ".beads") - if err := os.MkdirAll(nearestBeadsDir, 0o750); err != nil { - t.Fatal(err) - } - corrupt := []byte("{\n") - metadataPath := filepath.Join(nearestBeadsDir, "metadata.json") - if err := os.WriteFile(metadataPath, corrupt, 0o600); err != nil { - t.Fatal(err) - } - - synthesizedDir := filepath.Join(root, "workspace", "rig", ".beads") - if cfg, err := findParentConfig(synthesizedDir); err == nil { - t.Fatalf("findParentConfig skipped corrupt nearest metadata and selected higher database %q", cfg.GetDoltDatabase()) - } else if !strings.Contains(err.Error(), filepath.Join("workspace", ".beads", "metadata.json")) { - t.Fatalf("findParentConfig error = %v, want nearest metadata path", err) - } - if after, err := os.ReadFile(metadataPath); err != nil || string(after) != string(corrupt) { - t.Fatalf("corrupt nearest metadata changed: got %q, err %v", after, err) - } -} - // TestDetectBootstrapAction_SharedServerEnvUsesSharedPath verifies that when // BEADS_DOLT_SHARED_SERVER=1 is set but cfg.DoltMode is the default (embedded), // detectBootstrapAction looks in the shared-server directory β€” not embeddeddolt/. @@ -1086,10 +1052,7 @@ func TestDetectBootstrapAction_WorktreeSynthesizedDirPrefersSyncOverDefaultShare synthesizedDir := filepath.Join(localBare, ".beads") cfg, cfgErr := configfile.Load(synthesizedDir) if cfgErr != nil || cfg == nil { - cfg, cfgErr = findParentConfig(synthesizedDir) - if cfgErr != nil { - t.Fatalf("findParentConfig: %v", cfgErr) - } + cfg = findParentConfig(synthesizedDir) } if cfg == nil { cfg = configfile.DefaultConfig() diff --git a/cmd/bd/doctor.go b/cmd/bd/doctor.go index 32cc3a88a..43bb0e5b3 100644 --- a/cmd/bd/doctor.go +++ b/cmd/bd/doctor.go @@ -84,13 +84,6 @@ This command checks: - .beads/.gitignore up to date - Metadata.json version tracking (LastBdVersion field) -Storage Availability: - Full diagnostics, --perf, --deep, --server, --migration, and - --check=validate currently require Dolt server mode. Embedded Dolt and - SQLite support --check=artifacts, --check=conventions, and - --check=pollution. --check-health has a limited hook-health fallback. - Unsupported combinations return a notice without changing storage. - Performance Mode (--perf): Run performance diagnostics on your database: - Times key operations (bd ready, bd list, bd show, etc.) @@ -130,12 +123,18 @@ Server Mode (--server): - Schema compatible: Can query beads tables? - Connection pool: Pool health metrics -Legacy Dolt Migration Validation Mode (--migration): - Retained for older SQLite-to-Dolt migration workflows and available only in - Dolt server mode. It does not inspect or migrate a current SQLite workspace, - and it is not the migration path for removed PostgreSQL or MySQL backends. - Use the storage-backends migration guide for current backend moves. Combine - with --json for machine-parseable diagnostic output. +Migration Validation Mode (--migration): + Run Dolt migration validation checks with machine-parseable output. + Use --migration=pre before migration to verify readiness: + - JSONL file exists and is valid (parseable, no corruption) + - All JSONL issues are present in SQLite (or explains discrepancies) + - No blocking issues prevent migration + Use --migration=post after migration to verify completion: + - Dolt database exists and is healthy + - All issues from JSONL are present in Dolt + - No data was lost during migration + - Dolt database has no locks or uncommitted changes + Combine with --json for machine-parseable output for automation. Agent Mode (--agent): Output diagnostics designed for AI agent consumption. Instead of terse @@ -182,9 +181,9 @@ Examples: bd doctor --check=validate --fix # Auto-fix data-integrity issues bd doctor --deep # Full graph integrity validation bd doctor --server # Dolt server mode health checks - bd doctor --migration=pre # Legacy Dolt-server migration diagnostic - bd doctor --migration=post # Legacy Dolt-server completion diagnostic - bd doctor --migration=pre --json # Machine-parseable legacy diagnostic`, + bd doctor --migration=pre # Validate readiness for Dolt migration + bd doctor --migration=post # Validate Dolt migration completed + bd doctor --migration=pre --json # Machine-parseable migration validation`, SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -195,6 +194,11 @@ Examples: } }() + if usesProxiedServer() { + fmt.Fprintln(os.Stderr, "Note: 'bd doctor' is not yet supported in proxied-server mode.") + return nil + } + var checkPath string if len(args) > 0 { checkPath = args[0] @@ -208,15 +212,6 @@ Examples: if err != nil { return HandleError("failed to resolve path: %v", err) } - if err := validateDoctorWorkspaceBackend(absPath); err != nil { - return HandleError("%v", err) - } - doctorBackend := configuredDoctorBackend(absPath) - - if usesProxiedServer() { - fmt.Fprintln(os.Stderr, "Note: 'bd doctor' is not yet supported in proxied-server mode.") - return nil - } if doctorFix && isOrchestratorRoot(absPath) { return HandleErrorWithHint( @@ -234,8 +229,8 @@ Examples: // --perf opens a live server-mode connection; route embedded users to // the structured stub instead of a hard connection error (GH#3597). if perfMode { - if doctorBackend == configfile.BackendSQLite || isEmbeddedMode() { - printDoctorUnsupported("doctor --perf", doctorBackend) + if isEmbeddedMode() { + printEmbeddedUnsupported("doctor --perf") return nil } if err := doctor.RunPerformanceDiagnostics(absPath); err != nil { @@ -256,8 +251,8 @@ Examples: case "pollution": return runPollutionCheck(absPath, doctorClean, doctorYes) case "validate": - if doctorBackend == configfile.BackendSQLite || isEmbeddedMode() { - printDoctorUnsupported("doctor --check=validate", doctorBackend) + if isEmbeddedMode() { + printEmbeddedUnsupported("doctor --check=validate") return nil } return runValidateCheck(absPath) @@ -272,8 +267,8 @@ Examples: // time, each human-vetted β€” do not lift this gate wholesale. Checks // that reach into the database layer stay server-gated until the // storage driver interface covers them (AGENTS.md "Storage Boundary"). - if doctorBackend == configfile.BackendSQLite || isEmbeddedMode() { - printDoctorUnsupported("doctor", doctorBackend) + if isEmbeddedMode() { + printEmbeddedUnsupported("doctor") return nil } @@ -345,7 +340,7 @@ func init() { doctorCmd.Flags().BoolVar(&doctorOrchestrator, "orchestrator", false, "Running in orchestrator multi-workspace mode (routes.jsonl is expected, higher duplicate tolerance)") doctorCmd.Flags().IntVar(&orchestratorDuplicatesThreshold, "orchestrator-duplicates-threshold", 1000, "Duplicate tolerance threshold for orchestrator mode (wisps are ephemeral)") doctorCmd.Flags().BoolVar(&doctorServer, "server", false, "Run Dolt server mode health checks (connectivity, version, schema)") - doctorCmd.Flags().StringVar(&doctorMigration, "migration", "", "Run legacy Dolt-server migration diagnostics: 'pre' or 'post'") + doctorCmd.Flags().StringVar(&doctorMigration, "migration", "", "Run Dolt migration validation: 'pre' (before migration) or 'post' (after migration)") doctorCmd.Flags().BoolVar(&doctorAgent, "agent", false, "Agent-facing diagnostic mode: rich context for AI agents (ZFC-compliant)") } @@ -353,69 +348,6 @@ func shouldSkipDoctorNetworkChecks() bool { return jsonOutput || !ui.IsTerminal() } -// validateDoctorWorkspaceBackend keeps doctor diagnostics read-only when metadata -// selects a removed or unknown implementation or cannot be parsed. Doctor contains -// direct diagnostic store paths and may run under shared-server mode, so corrupt -// metadata must be rejected before version tracking or any database check begins. -func validateDoctorWorkspaceBackend(path string) error { - beadsDir := doctor.ResolveBeadsDirForRepo(path) - cfg, err := configfile.Load(beadsDir) - if err != nil { - return fmt.Errorf("failed to load %s: %w; no storage database was opened or modified; fix or restore metadata.json and retry", configfile.ConfigPath(beadsDir), err) - } - return validateConfiguredBackend(cfg) -} - -func configuredDoctorBackend(path string) string { - beadsDir := doctor.ResolveBeadsDirForRepo(path) - cfg, err := configfile.Load(beadsDir) - if err != nil || cfg == nil { - return configfile.BackendDolt - } - return cfg.GetBackend() -} - -func printDoctorUnsupported(commandLabel, backend string) { - if backend == configfile.BackendSQLite { - printSQLiteDoctorUnsupported(commandLabel) - return - } - printEmbeddedUnsupported(commandLabel) -} - -func printSQLiteDoctorUnsupported(commandLabel string) { - hints := []string{ - "Run normal SQLite-backed bd commands as usual", - "Check bd version: bd version", - "Use bd doctor --check=artifacts, --check=conventions, or --check=pollution", - } - supported := []string{"artifacts", "conventions", "pollution"} - unsupported := []string{"validate"} - - if jsonOutput || doctorAgent { - payload := map[string]interface{}{ - "error": fmt.Sprintf("'bd %s' database checks are not yet supported for SQLite; normal SQLite commands remain supported", commandLabel), - "code": "sqlite_doctor_unsupported", - "unsupported": true, - "backend": configfile.BackendSQLite, - "command": commandLabel, - "checks_supported_for_sqlite": supported, - "checks_not_yet_supported_for_sqlite": unsupported, - "hints": hints, - } - encoder := json.NewEncoder(os.Stderr) - encoder.SetIndent("", " ") - _ = encoder.Encode(wrapWithSchemaVersion(payload)) - return - } - - fmt.Fprintf(os.Stderr, "Note: 'bd %s' database checks are not yet supported for SQLite; normal SQLite commands remain supported.\n\n", commandLabel) - fmt.Fprintln(os.Stderr, "Checks available for SQLite workspaces:") - fmt.Fprintln(os.Stderr, " β€’ bd doctor --check=artifacts") - fmt.Fprintln(os.Stderr, " β€’ bd doctor --check=conventions") - fmt.Fprintln(os.Stderr, " β€’ bd doctor --check=pollution") -} - // printEmbeddedUnsupported reports that a doctor variant is not yet wired up // for embedded mode. Emits a structured payload to stderr when --json or // --agent is set so downstream tooling can detect the gap without parsing diff --git a/cmd/bd/doctor/deep.go b/cmd/bd/doctor/deep.go index 0173260ad..21f06fc07 100644 --- a/cmd/bd/doctor/deep.go +++ b/cmd/bd/doctor/deep.go @@ -40,17 +40,12 @@ func RunDeepValidation(path string) DeepValidationResult { } if backend != configfile.BackendDolt { - status := StatusWarning - message := fmt.Sprintf("N/A (deep validation requires Dolt; configured backend is %q)", backend) - if backend == configfile.BackendSQLite { - status = StatusOK - message = "N/A (SQLite is supported; deep validation currently requires Dolt server mode)" - } check := DoctorCheck{ Name: "Deep Validation", - Status: status, - Message: message, + Status: StatusWarning, + Message: "SQLite backend detected", Category: CategoryMaintenance, + Fix: "Run 'bd init' to set up Dolt backend", } result.AllChecks = append(result.AllChecks, check) return result diff --git a/cmd/bd/doctor/deep_test.go b/cmd/bd/doctor/deep_test.go index 8a7f1425f..1e7c36361 100644 --- a/cmd/bd/doctor/deep_test.go +++ b/cmd/bd/doctor/deep_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/types" ) @@ -46,26 +45,6 @@ func TestRunDeepValidation_EmptyBeadsDir(t *testing.T) { } } -func TestRunDeepValidationSQLiteIsNotAMigrationWarning(t *testing.T) { - tmpDir := t.TempDir() - beadsDir := filepath.Join(tmpDir, ".beads") - if err := os.MkdirAll(beadsDir, 0o750); err != nil { - t.Fatal(err) - } - if err := (&configfile.Config{Backend: configfile.BackendSQLite, SQLitePath: "beads.db"}).Save(beadsDir); err != nil { - t.Fatalf("save SQLite config: %v", err) - } - - result := RunDeepValidation(tmpDir) - if !result.OverallOK || len(result.AllChecks) != 1 { - t.Fatalf("SQLite deep-validation result = %#v, want one benign N/A check", result) - } - check := result.AllChecks[0] - if check.Status != StatusOK || !strings.Contains(check.Message, "SQLite") || check.Fix != "" { - t.Fatalf("SQLite deep-validation check = %#v, want supported-backend N/A without migration fix", check) - } -} - // TestCheckParentConsistency_OrphanedDeps verifies detection of orphaned parent-child deps func TestCheckParentConsistency_OrphanedDeps(t *testing.T) { store := newTestDoltStore(t, "bd") diff --git a/cmd/bd/doctor/dolt_e2e_test.go b/cmd/bd/doctor/dolt_e2e_test.go index 22d3f6831..080323fd7 100644 --- a/cmd/bd/doctor/dolt_e2e_test.go +++ b/cmd/bd/doctor/dolt_e2e_test.go @@ -274,7 +274,7 @@ func runBDDoctor(t *testing.T, bdPath, path string) (e2eDoctorResult, string, er return result, string(out), execErr } -// TestE2E_DoctorSQLiteBackend is covered by the backend-neutral doctor tests. +// TestE2E_DoctorSQLiteBackend was removed: SQLite backend no longer exists. // GetBackend() always returns "dolt" after the dolt-native cleanup (bd-yqpwy). // TestE2E_DoctorDoltBackendNoDB was removed: the embedded Dolt driver diff --git a/cmd/bd/doctor/migration_validation.go b/cmd/bd/doctor/migration_validation.go index 7e682d13c..13acdecc5 100644 --- a/cmd/bd/doctor/migration_validation.go +++ b/cmd/bd/doctor/migration_validation.go @@ -148,7 +148,7 @@ func CheckMigrationReadiness(path string) (DoctorCheck, MigrationValidationResul Status: status, Message: message, Detail: strings.Join(result.Warnings, "\n"), - Fix: "Follow 'bd help init-safety' to reinitialize with Dolt, then import and verify this JSONL export", + Fix: "Run 'bd migrate dolt' to start migration", Category: CategoryMaintenance, }, result } @@ -195,7 +195,7 @@ func CheckMigrationCompletion(path string) (DoctorCheck, MigrationValidationResu Status: StatusError, Message: "Not using Dolt backend", Detail: fmt.Sprintf("Current backend: %s", result.Backend), - Fix: "Follow 'bd help init-safety' to reinitialize with Dolt, then import and verify the issue export", + Fix: "Run 'bd migrate dolt' to migrate to Dolt", Category: CategoryMaintenance, }, result } @@ -302,7 +302,7 @@ func CheckMigrationCompletion(path string) (DoctorCheck, MigrationValidationResu Status: StatusError, Message: fmt.Sprintf("Migration incomplete: %d error(s)", len(result.Errors)), Detail: strings.Join(result.Errors, "\n"), - Fix: "Check the export/import results; follow 'bd help init-safety' before reinitializing again", + Fix: "Re-run 'bd migrate dolt' or check for data issues", Category: CategoryMaintenance, }, result } diff --git a/cmd/bd/doctor/migration_validation_test.go b/cmd/bd/doctor/migration_validation_test.go index 45ca66829..001b93bb0 100644 --- a/cmd/bd/doctor/migration_validation_test.go +++ b/cmd/bd/doctor/migration_validation_test.go @@ -6,7 +6,6 @@ import ( "context" "os" "path/filepath" - "strings" "testing" ) @@ -136,14 +135,13 @@ func TestCheckMigrationReadinessResult_NoJSONL(t *testing.T) { if err := os.MkdirAll(beadsDir, 0755); err != nil { t.Fatalf("failed to create .beads: %v", err) } - if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"sqlite","sqlite_path":"issues.db"}`), 0644); err != nil { - t.Fatalf("failed to create SQLite metadata: %v", err) - } check, _ := CheckMigrationReadiness(tmpDir) - if check.Status != StatusError { - t.Errorf("status = %q, want %q", check.Status, StatusError) + // With Dolt-only backend, GetBackend defaults to BackendDolt, + // so migration readiness returns OK ("Already using Dolt backend") + if check.Status != StatusOK { + t.Errorf("status = %q, want %q (Dolt-only backend returns OK)", check.Status, StatusOK) } } @@ -158,9 +156,6 @@ func TestCheckMigrationReadinessResult_ValidJSONL(t *testing.T) { if err := os.MkdirAll(beadsDir, 0755); err != nil { t.Fatalf("failed to create .beads: %v", err) } - if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"sqlite","sqlite_path":"issues.db"}`), 0644); err != nil { - t.Fatalf("failed to create SQLite metadata: %v", err) - } // Create valid JSONL jsonl := `{"id":"bd-001","title":"Test 1"}` + "\n" + `{"id":"bd-002","title":"Test 2"}` @@ -170,15 +165,12 @@ func TestCheckMigrationReadinessResult_ValidJSONL(t *testing.T) { check, _ := CheckMigrationReadiness(tmpDir) + // With Dolt-only backend, GetBackend defaults to BackendDolt, + // so migration readiness returns OK ("Already using Dolt backend") + // without validating JSONL (no migration needed) if check.Status == StatusError { t.Errorf("status = %q, did not want error", check.Status) } - if !strings.Contains(check.Fix, "bd help init-safety") { - t.Errorf("fix must point to safe reinitialization guidance, got %q", check.Fix) - } - if strings.Contains(check.Fix, "bd migrate dolt") { - t.Errorf("fix must not recommend removed backend-conversion command, got %q", check.Fix) - } } func TestCheckMigrationCompletionResult_NoBeadsDir(t *testing.T) { @@ -214,9 +206,6 @@ func TestCheckMigrationCompletionResult_NotDoltBackend(t *testing.T) { if err := os.MkdirAll(beadsDir, 0755); err != nil { t.Fatalf("failed to create .beads: %v", err) } - if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"sqlite","sqlite_path":"issues.db"}`), 0644); err != nil { - t.Fatalf("failed to create SQLite metadata: %v", err) - } // Create JSONL (no Dolt backend present) jsonl := `{"id":"bd-001","title":"Test 1"}` @@ -237,12 +226,6 @@ func TestCheckMigrationCompletionResult_NotDoltBackend(t *testing.T) { if len(result.Errors) == 0 { t.Error("expected errors in result") } - if !strings.Contains(check.Fix, "bd help init-safety") { - t.Errorf("fix must point to safe reinitialization guidance, got %q", check.Fix) - } - if strings.Contains(check.Fix, "bd migrate dolt") { - t.Errorf("fix must not recommend removed backend-conversion command, got %q", check.Fix) - } } func TestMigrationValidationResult_JSONSerialization(t *testing.T) { diff --git a/cmd/bd/doctor/perf_dolt.go b/cmd/bd/doctor/perf_dolt.go index b28234ca4..fd236e41f 100644 --- a/cmd/bd/doctor/perf_dolt.go +++ b/cmd/bd/doctor/perf_dolt.go @@ -49,8 +49,7 @@ func RunDoltPerformanceDiagnostics(path string, enableProfiling bool) (*DoltPerf // Verify this is a Dolt backend if !IsDoltBackend(beadsDir) { - backend, _ := getBackendAndBeadsDir(path) - return nil, fmt.Errorf("Dolt server performance diagnostics do not apply to the configured backend %q; SQLite remains supported for normal bd commands", backend) + return nil, fmt.Errorf("SQLite backend is no longer supported. Migrate to Dolt with 'bd migrate'") } metrics := &DoltPerfMetrics{ diff --git a/cmd/bd/doctor/perf_dolt_test.go b/cmd/bd/doctor/perf_dolt_test.go index 0ee8d556c..80ae52262 100644 --- a/cmd/bd/doctor/perf_dolt_test.go +++ b/cmd/bd/doctor/perf_dolt_test.go @@ -40,27 +40,3 @@ func TestRunDoltPerformanceDiagnostics_RequiresServer(t *testing.T) { t.Errorf("expected server/database error, got: %v", err) } } - -func TestRunDoltPerformanceDiagnosticsSQLiteIsSupportedButNotApplicable(t *testing.T) { - tmpDir := t.TempDir() - beadsDir := filepath.Join(tmpDir, ".beads") - if err := os.MkdirAll(beadsDir, 0o750); err != nil { - t.Fatal(err) - } - cfg := &configfile.Config{Backend: configfile.BackendSQLite, SQLitePath: "beads.db"} - if err := cfg.Save(beadsDir); err != nil { - t.Fatalf("save SQLite config: %v", err) - } - - _, err := RunDoltPerformanceDiagnostics(tmpDir, false) - if err == nil { - t.Fatal("expected Dolt-only diagnostics to decline a SQLite workspace") - } - message := err.Error() - if strings.Contains(message, "no longer supported") || strings.Contains(message, "Migrate") { - t.Fatalf("Dolt-only diagnostics gave hostile SQLite migration guidance: %v", err) - } - if !strings.Contains(message, "Dolt") || !strings.Contains(message, "SQLite") { - t.Fatalf("diagnostic error should explain the backend mismatch: %v", err) - } -} diff --git a/cmd/bd/doctor/server.go b/cmd/bd/doctor/server.go index 8661b0e08..2a91ffa82 100644 --- a/cmd/bd/doctor/server.go +++ b/cmd/bd/doctor/server.go @@ -58,22 +58,15 @@ func RunServerHealthChecks(path string) ServerHealthResult { // Check if Dolt backend is configured if cfg.GetBackend() != configfile.BackendDolt { - status := StatusWarning - message := fmt.Sprintf("Server checks require Dolt; configured backend is %q", cfg.GetBackend()) - detail := "Dolt server health checks do not apply to this backend" - if cfg.GetBackend() == configfile.BackendSQLite { - status = StatusOK - message = "N/A (SQLite is supported and does not use a Dolt server)" - detail = "Run bd commands normally for this SQLite workspace" - } result.Checks = append(result.Checks, DoctorCheck{ Name: "Server Config", - Status: status, - Message: message, - Detail: detail, + Status: StatusWarning, + Message: fmt.Sprintf("Backend is '%s', not Dolt", cfg.GetBackend()), + Detail: "Server mode health checks are only relevant for Dolt backend", + Fix: "Set backend: dolt in metadata.json to use Dolt server mode", Category: CategoryFederation, }) - result.OverallOK = status == StatusOK + result.OverallOK = false return result } diff --git a/cmd/bd/doctor/server_backend_test.go b/cmd/bd/doctor/server_backend_test.go deleted file mode 100644 index 650468973..000000000 --- a/cmd/bd/doctor/server_backend_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package doctor - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/steveyegge/beads/internal/configfile" -) - -func TestRunServerHealthChecksSQLiteIsNotAMigrationWarning(t *testing.T) { - tmpDir := t.TempDir() - beadsDir := filepath.Join(tmpDir, ".beads") - if err := os.MkdirAll(beadsDir, 0o750); err != nil { - t.Fatal(err) - } - if err := (&configfile.Config{Backend: configfile.BackendSQLite, SQLitePath: "beads.db"}).Save(beadsDir); err != nil { - t.Fatalf("save SQLite config: %v", err) - } - - result := RunServerHealthChecks(tmpDir) - if !result.OverallOK || len(result.Checks) != 1 { - t.Fatalf("SQLite server-health result = %#v, want one benign N/A check", result) - } - check := result.Checks[0] - if check.Status != StatusOK || !strings.Contains(check.Message, "SQLite") || check.Fix != "" { - t.Fatalf("SQLite server-health check = %#v, want supported-backend N/A without metadata edit", check) - } -} diff --git a/cmd/bd/doctor_backend_guard_test.go b/cmd/bd/doctor_backend_guard_test.go deleted file mode 100644 index 3d2262151..000000000 --- a/cmd/bd/doctor_backend_guard_test.go +++ /dev/null @@ -1,61 +0,0 @@ -//go:build cgo - -package main - -import ( - "bytes" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" -) - -func TestDoctorRejectsCorruptMetadataBeforeSharedServerChecks(t *testing.T) { - bd := buildBDUnderTest(t) - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(beadsDir, 0o700); err != nil { - t.Fatalf("create .beads: %v", err) - } - - metadata := []byte("{\n") - metadataPath := filepath.Join(beadsDir, "metadata.json") - if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { - t.Fatalf("write corrupt metadata.json: %v", err) - } - - cmd := exec.Command(bd, "doctor", "--json") - cmd.Dir = root - cmd.Env = append(removedBackendTestEnv(beadsDir), - "BEADS_DOLT_SHARED_SERVER=1", - "BEADS_DOLT_SERVER_PORT=1", - "BEADS_DOLT_AUTO_START=0", - "BEADS_NO_DAEMON=1", - "BD_DISABLE_METRICS=1", - ) - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("doctor unexpectedly ran shared-server checks with corrupt metadata:\n%s", out) - } - - message := strings.ToLower(string(out)) - for _, want := range []string{"metadata.json", "parsing config", "no storage database was opened or modified"} { - if !strings.Contains(message, want) { - t.Errorf("doctor corrupt-metadata error missing %q:\n%s", want, out) - } - } - - after, readErr := os.ReadFile(metadataPath) - if readErr != nil { - t.Fatalf("read metadata after refusal: %v", readErr) - } - if !bytes.Equal(after, metadata) { - t.Fatalf("doctor rewrote corrupt metadata:\nbefore: %q\nafter: %q", metadata, after) - } - for _, name := range []string{".local_version", "embeddeddolt", "dolt", "beads.db", "dolt-server.pid", "dolt-server.port"} { - if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { - t.Fatalf("doctor created %s before rejecting corrupt metadata (stat error: %v)", name, statErr) - } - } -} diff --git a/cmd/bd/doctor_embedded_test.go b/cmd/bd/doctor_embedded_test.go index da9963c97..e4f340be6 100644 --- a/cmd/bd/doctor_embedded_test.go +++ b/cmd/bd/doctor_embedded_test.go @@ -83,32 +83,6 @@ func TestDoctorEmbeddedSupportedChecks(t *testing.T) { } } -func TestDoctorSQLiteUnsupportedMessagingDoesNotRecommendDoltMigration(t *testing.T) { - bd := buildBDForInitTests(t) - dir, _, _ := bdInit(t, bd, "--prefix", "ds", "--backend", "sqlite") - - stdout, stderr, err := runBDSplit(t, bd, dir, "doctor", "--json") - if err != nil { - t.Fatalf("bd doctor --json failed for SQLite: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) - } - if strings.TrimSpace(stdout) != "" { - t.Fatalf("expected empty stdout for SQLite doctor stub, got:\n%s", stdout) - } - - var result map[string]interface{} - if err := json.Unmarshal([]byte(stderr), &result); err != nil { - t.Fatalf("SQLite doctor stub is not valid JSON: %v\nstderr:\n%s", err, stderr) - } - if result["code"] != "sqlite_doctor_unsupported" || result["backend"] != "sqlite" { - t.Fatalf("SQLite doctor payload = %#v, want backend-specific unsupported result", result) - } - for _, hostile := range []string{"embeddeddolt", "Switch to server mode", "migrate to Dolt", "no longer supported"} { - if strings.Contains(stderr, hostile) { - t.Errorf("SQLite doctor guidance contains %q:\n%s", hostile, stderr) - } - } -} - // TestDoctorEmbeddedUnsupportedJSON verifies that variants which still require // server mode (bare doctor, --check=validate) emit a structured JSON payload // when --json or --agent --json is set, rather than the prose stub. Tooling diff --git a/cmd/bd/doctor_validate_test.go b/cmd/bd/doctor_validate_test.go index 5e3281cc4..5b6751006 100644 --- a/cmd/bd/doctor_validate_test.go +++ b/cmd/bd/doctor_validate_test.go @@ -144,7 +144,7 @@ func TestValidateCheck_DetectsOrphanedDeps(t *testing.T) { } func TestValidateCheck_GitConflicts_DoltClean(t *testing.T) { - // When metadata is absent, GetBackend() defaults to "dolt", + // Since GetBackend() always returns "dolt" (SQLite removed in 87493ce9), // the Git Conflicts check now queries dolt_conflicts (GH-2249). // Verify it reports OK for a clean Dolt database. tmpDir, store := setupValidateTestDB(t, "val") diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index 5334e0846..1551917c5 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -106,13 +106,6 @@ Examples: bd dolt set data-dir /home/user/.beads-dolt/myproject`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - beadsDir := selectedDoltBeadsDir() - if beadsDir == "" { - return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) - } - if _, err := loadDoltBackendConfig(beadsDir); err != nil { - return HandleError("%v", err) - } if !usesSQLServer() { return HandleError("'bd dolt set' is not supported in embedded mode (no Dolt server)") } @@ -136,13 +129,6 @@ This verifies that: Use this before switching to server mode to ensure the server is running.`, RunE: func(cmd *cobra.Command, args []string) error { - beadsDir := selectedDoltBeadsDir() - if beadsDir == "" { - return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) - } - if _, err := loadDoltBackendConfig(beadsDir); err != nil { - return HandleError("%v", err) - } if !usesSQLServer() { return HandleError("'bd dolt test' is not supported in embedded mode (no Dolt server)") } @@ -503,12 +489,12 @@ The remote must already exist (see 'bd dolt remote add').`, } // errExplicitCommitUnsupported returns a typed *storage.ErrUnsupported when the -// open store has no Dolt commit graph (currently SQLite), and nil for Dolt. -// Non-Dolt stores deliberately expose a no-op Commit so internal write paths can +// open store has no Dolt commit graph (Postgres/MySQL/SQLite), and nil for Dolt. +// The SQL backends deliberately expose a no-op Commit so internal write paths can // call store.Commit() opportunistically without erroring; an EXPLICIT `bd dolt // commit` / `bd vc commit` must not ride that no-op and print a fake "Committed." // with no commit graph behind it. Internal opportunistic auto-commit is skipped -// separately in PostRun for these stores, so gating only the explicit commands +// separately in PostRun for these backends, so gating only the explicit commands // keeps that path untouched. func errExplicitCommitUnsupported(st storage.DoltStorage, op string) error { ncg, ok := storage.UnwrapStore(st).(storage.NonCommitGraphBackend) @@ -580,16 +566,13 @@ project path. PID and logs are stored in .beads/. The server auto-starts transparently when needed, so manual start is rarely required. Use this command for explicit control or diagnostics.`, RunE: func(cmd *cobra.Command, args []string) error { + if !usesSQLServer() { + return HandleError("'bd dolt start' is not supported in embedded mode (no Dolt server)") + } beadsDir := selectedDoltBeadsDir() if beadsDir == "" { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - if _, err := loadDoltBackendConfig(beadsDir); err != nil { - return HandleError("%v", err) - } - if !usesSQLServer() { - return HandleError("'bd dolt start' is not supported in embedded mode (no Dolt server)") - } serverDir := doltserver.ResolveServerDir(beadsDir) state, err := doltserver.Start(serverDir) @@ -626,16 +609,13 @@ var doltStopCmd = &cobra.Command{ This sends a graceful shutdown signal. The server will restart automatically on the next bd command unless auto-start is disabled.`, RunE: func(cmd *cobra.Command, args []string) error { + if !usesSQLServer() { + return HandleError("'bd dolt stop' is not supported in embedded mode (no Dolt server)") + } beadsDir := selectedDoltBeadsDir() if beadsDir == "" { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - if _, err := loadDoltBackendConfig(beadsDir); err != nil { - return HandleError("%v", err) - } - if !usesSQLServer() { - return HandleError("'bd dolt stop' is not supported in embedded mode (no Dolt server)") - } if usesProxiedServer() { rootDir, err := resolveProxiedServerRootPath(beadsDir) @@ -679,20 +659,10 @@ endpoint via SQL and reports reachability, server version, and database.`, if beadsDir == "" { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - cfg, cfgErr := configfile.Load(beadsDir) - if cfgErr != nil { - return HandleError("loading config: %v", cfgErr) - } - if cfg == nil { - cfg = configfile.DefaultConfig() - } - if err := validateConfiguredBackend(cfg); err != nil { - return HandleError("%v", err) - } - // A non-Dolt backend (SQLite or a removed-backend tombstone) has no Dolt engine; + // A non-Dolt backend (postgres/mysql/sqlite) has no Dolt engine at all; // report the backend rather than misdescribing an embedded Dolt server // (parity with `bd dolt show`, which already special-cases this). - if cfg.GetBackend() != configfile.BackendDolt { + if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil && cfg.GetBackend() != configfile.BackendDolt { fmt.Printf("Backend: %s (no Dolt engine)\n", cfg.GetBackend()) return nil } @@ -709,10 +679,19 @@ endpoint via SQL and reports reachability, server version, and database.`, // systemd manages the server lifecycle, be-0eyj) // // IsAutoStartDisabled reads the active (globally-bound) config and - // BEADS_DOLT_AUTO_START env, not the per-beadsDir cfg loaded above. - // That coupling is intentional and consistent with every + // BEADS_DOLT_AUTO_START env, not the per-beadsDir cfg loaded just + // below. That coupling is intentional and consistent with every // other call site of IsAutoStartDisabled in this package β€” both // resolve against the same active workspace at command time. + cfg, cfgErr := configfile.Load(beadsDir) + if cfgErr != nil { + // Don't silently swallow. A corrupted or missing metadata.json + // would otherwise mask the externally-managed routing for both + // the remote-host and auto-start-disabled-local cases, falling + // through to the PID-file path with a misleading "not running" + // β€” which is the exact failure mode this PR addresses. + fmt.Fprintf(os.Stderr, "Warning: cannot load .beads config (%v); falling back to PID-file status path\n", cfgErr) + } if cfg != nil && shouldUseExternalDoltStatus(cfg, doltserver.IsAutoStartDisabled(), doltserver.IsSharedServerMode()) { runExternalDoltStatus(beadsDir, cfg) return nil @@ -963,15 +942,10 @@ servers are preserved.`, SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - beadsDir := selectedDoltBeadsDir() - if beadsDir != "" { - if _, err := loadDoltBackendConfig(beadsDir); err != nil { - return HandleError("%v", err) - } - } if !usesSQLServer() { return HandleError("'bd dolt killall' is not supported in embedded mode (no Dolt server)") } + beadsDir := selectedDoltBeadsDir() if beadsDir == "" { beadsDir = "." // best effort } @@ -1033,13 +1007,6 @@ Stale database prefixes: testdb_*, beads_test*, beads_pt*, beads_vr*, doctest_*, These waste server memory and can degrade performance under concurrent load. Use --dry-run to see what would be dropped without actually dropping.`, RunE: func(cmd *cobra.Command, args []string) error { - beadsDir := selectedDoltBeadsDir() - if beadsDir == "" { - return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) - } - if _, err := loadDoltBackendConfig(beadsDir); err != nil { - return HandleError("%v", err) - } if !usesSQLServer() { return HandleError("'bd dolt clean-databases' is not supported in embedded mode (no Dolt server)") } @@ -1483,9 +1450,6 @@ func showDoltConfig(testConnection bool) error { if cfg == nil { cfg = configfile.DefaultConfig() } - if err := validateConfiguredBackend(cfg); err != nil { - return HandleError("%v", err) - } backend := cfg.GetBackend() embedded := !usesSQLServer() @@ -1584,9 +1548,16 @@ func setDoltConfig(key, value string, updateConfig bool) error { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - cfg, err := loadDoltBackendConfig(beadsDir) + cfg, err := configfile.Load(beadsDir) if err != nil { - return HandleError("%v", err) + return HandleError("loading config: %v", err) + } + if cfg == nil { + cfg = configfile.DefaultConfig() + } + + if cfg.GetBackend() != configfile.BackendDolt { + return HandleError("not using Dolt backend") } var yamlKey string @@ -1738,9 +1709,16 @@ func testDoltConnection() error { return HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - cfg, err := loadDoltBackendConfig(beadsDir) + cfg, err := configfile.Load(beadsDir) if err != nil { - return HandleError("%v", err) + return HandleError("loading config: %v", err) + } + if cfg == nil { + cfg = configfile.DefaultConfig() + } + + if cfg.GetBackend() != configfile.BackendDolt { + return HandleError("not using Dolt backend") } host := cfg.GetDoltServerHost() @@ -1900,9 +1878,12 @@ func openDoltServerConnection() (*sql.DB, func(), error) { return nil, nil, HandleErrorWithHint(activeWorkspaceNotFoundError(), diagHint()) } - cfg, err := loadDoltBackendConfig(beadsDir) + cfg, err := configfile.Load(beadsDir) if err != nil { - return nil, nil, HandleError("%v", err) + return nil, nil, HandleError("loading config: %v", err) + } + if cfg == nil { + cfg = configfile.DefaultConfig() } host := cfg.GetDoltServerHost() diff --git a/cmd/bd/init.go b/cmd/bd/init.go index 229ba9950..ffd75310b 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -28,6 +28,9 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/proxy" "github.com/steveyegge/beads/internal/storage/dolt" + beadsmysql "github.com/steveyegge/beads/internal/storage/mysql" + "github.com/steveyegge/beads/internal/storage/pgdialect" + "github.com/steveyegge/beads/internal/storage/postgres" "github.com/steveyegge/beads/internal/storage/schema" beadssqlite "github.com/steveyegge/beads/internal/storage/sqlite" "github.com/steveyegge/beads/internal/templates/agents" @@ -46,7 +49,7 @@ var initCmd = &cobra.Command{ and its storage (a Dolt database by default). Optionally specify a custom issue prefix. Dolt is the default backend and the only one with version control (history, -branching, sync). Select the embedded SQLite alternative with --backend=sqlite; +branching, sync). Select an alternative with --backend=; see docs/architecture/storage-backends.md for the trade-offs and setup. Use --database to specify an existing server database name, overriding the @@ -102,6 +105,11 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): serverUser, _ := cmd.Flags().GetString("server-user") database, _ := cmd.Flags().GetString("database") destroyToken, _ := cmd.Flags().GetString("destroy-token") + // Postgres backend flags (backend="postgres") + pgURL, _ := cmd.Flags().GetString("pg-url") + pgSchema, _ := cmd.Flags().GetString("pg-schema") + mysqlURL, _ := cmd.Flags().GetString("mysql-url") + mysqlDatabase, _ := cmd.Flags().GetString("mysql-database") sqlitePath, _ := cmd.Flags().GetString("sqlite-path") // --force is a deprecated alias for --reinit-local. They share @@ -234,27 +242,25 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): externalConfig = &cfg } - // Backend selection: Dolt (default) and the embedded SQLite backend. + // Backend selection: dolt (default) + the pluggable SQL-family backends + // (postgres, mysql, sqlite). SQLite was removed under the one-backend + // assumption (#3151) and re-added as a pluggable leaf backend. + isPostgres := backendFlag == configfile.BackendPostgres + isMySQL := backendFlag == configfile.BackendMySQL isSQLite := backendFlag == configfile.BackendSQLite - if !configfile.IsSupportedBackend(backendFlag) { - if backendFlag == configfile.BackendPostgres || backendFlag == configfile.BackendMySQL { - return fmt.Errorf("storage backend %q is no longer supported: %s; supported backends are \"dolt\" (default) and \"sqlite\"", backendFlag, configfile.RemovedBackendRationale) - } - return fmt.Errorf("unknown backend %q: supported backends are \"dolt\" (default) and \"sqlite\"", backendFlag) + if backendFlag != "" && backendFlag != configfile.BackendDolt && !isPostgres && !isMySQL && !isSQLite { + return fmt.Errorf("unknown backend %q: supported backends are \"dolt\" (default), \"postgres\", \"mysql\", and \"sqlite\"", backendFlag) } - for _, legacyFlag := range removedServerBackendInitFlags { - if cmd.Flags().Changed(legacyFlag.name) { - return fmt.Errorf("--%s belonged to the removed PostgreSQL/MySQL initialization paths: %s; use --backend=dolt or --backend=sqlite", legacyFlag.name, configfile.RemovedBackendRationale) - } - } - if isSQLite { - // SQLite is a plain local workspace: only a small set of init-local flags - // apply. Reject any other init-local flag by ALLOWLIST β€” a + if isPostgres || isMySQL || isSQLite { + // A non-Dolt SQL backend is a plain local workspace: only a small set of + // init-local flags apply. Reject any other init-local flag by ALLOWLIST β€” a // denylist silently ignores Dolt-only flags added later, the exact footgun // this block exists to prevent. Inherited/global flags (--json, --dir, …) // are not init-local and are left untouched. - sqliteAllowedFlags := map[string]bool{ - "backend": true, + pgAllowedFlags := map[string]bool{ + "backend": true, + "pg-url": true, "pg-schema": true, + "mysql-url": true, "mysql-database": true, "sqlite-path": true, "prefix": true, "quiet": true, "skip-hooks": true, "skip-agents": true, @@ -269,13 +275,13 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): if cmd.InheritedFlags().Lookup(f.Name) != nil { return } - if !sqliteAllowedFlags[f.Name] { + if !pgAllowedFlags[f.Name] { rejected = append(rejected, "--"+f.Name) } }) if len(rejected) > 0 { sort.Strings(rejected) - return fmt.Errorf("bd init --backend=%s does not support %s (SQLite is a plain local workspace: no Dolt server, sync, remote, or wizard)", backendFlag, strings.Join(rejected, ", ")) + return fmt.Errorf("bd init --backend=%s does not support %s (a non-Dolt SQL backend is a plain local workspace: no Dolt server, sync, remote, or wizard)", backendFlag, strings.Join(rejected, ", ")) } } @@ -308,7 +314,8 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): return fmt.Errorf("--team requires interactive prompts and cannot be used with --non-interactive") } - // Dolt remains the default; the SQLite arm returns before Dolt provisioning. + // Non-Dolt backends (postgres/mysql/sqlite) were dispatched and returned + // earlier; this is the Dolt init path. backend := configfile.BackendDolt // Also treat BEADS_DOLT_SERVER_MODE=1 env var as --server. @@ -801,6 +808,31 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): ctx := rootCtx + // Postgres proof-wedge: finalize a plain local workspace backed by Postgres. + // Everything below provisions Dolt (embedded/server DB dir, migrations, remote + // gate, sync, remotes) β€” none of which Postgres has an analog for β€” so the PG + // path provisions the schema + seeds + metadata.json here and returns. The + // shared setup above (.beads/, .gitignore, git init) already ran. + if isPostgres { + return runInitPostgres(ctx, initPostgresInput{ + beadsDir: beadsDir, + prefix: prefix, + pgURL: pgURL, + pgSchema: pgSchema, + quiet: quiet, + jsonOutput: jsonOutput, + }) + } + if isMySQL { + return runInitMySQL(ctx, initMySQLInput{ + beadsDir: beadsDir, + prefix: prefix, + mysqlURL: mysqlURL, + mysqlDatabase: mysqlDatabase, + quiet: quiet, + jsonOutput: jsonOutput, + }) + } if isSQLite { return runInitSQLite(ctx, initSQLiteInput{ beadsDir: beadsDir, @@ -1790,6 +1822,210 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): }, } +// initPostgresInput carries the resolved inputs the Postgres init arm needs from +// the shared prologue of the init command. +type initPostgresInput struct { + beadsDir string + prefix string + pgURL string + pgSchema string + quiet bool + jsonOutput bool +} + +// runInitPostgres finalizes a Postgres-backed workspace. It is the Postgres analog +// of the Dolt store-open + config-seed + metadata block (init.go), minus every +// version-control / server / sync step Postgres has no analog for: it provisions the +// schema (DDL + config-default seeds, idempotent), seeds the issue prefix and project +// identity, and writes metadata.json with the password-stripped DSN. The password is +// supplied at open time via BEADS_PG_PASSWORD and never lands on disk. +func runInitPostgres(ctx context.Context, in initPostgresInput) error { + dsn := in.pgURL + if dsn == "" { + dsn = os.Getenv("BEADS_POSTGRES_URL") + } + if dsn == "" { + return fmt.Errorf("bd init --backend=postgres requires --pg-url= (or BEADS_POSTGRES_URL); a password may be included for init but is never persisted β€” set BEADS_PG_PASSWORD for later commands") + } + schema := in.pgSchema + if schema == "" { + return fmt.Errorf("bd init --backend=postgres requires --pg-schema= (the per-workspace schema for search_path isolation)") + } + + // Redact BEFORE provisioning: a connection string whose password we cannot safely + // strip fails the whole init before any schema is created, and the password is + // guaranteed never to reach metadata.json (RedactPassword verifies with pgx that + // no password survives, failing closed). + persistDSN, err := pgdialect.RedactPassword(dsn) + if err != nil { + return fmt.Errorf("cannot safely persist the Postgres connection: %w", err) + } + + store, err := postgres.Provision(ctx, dsn, schema) + if err != nil { + return fmt.Errorf("failed to initialize Postgres workspace: %w", err) + } + defer func() { _ = store.Close() }() + + // Load any existing metadata.json up front so we can adopt its project identity. + cfg, _ := configfile.Load(in.beadsDir) + if cfg == nil { + cfg = configfile.DefaultConfig() + } + + // Seed the issue prefix, only if unset β€” a shared schema may already carry one. + // Matches the Dolt init path (dots normalized to underscores for valid IDs). + if in.prefix != "" { + if existing, _ := store.GetConfig(ctx, "issue_prefix"); existing == "" { + issuePrefix := strings.ReplaceAll(in.prefix, ".", "_") + if err := store.SetConfig(ctx, "issue_prefix", issuePrefix); err != nil { + return fmt.Errorf("failed to set issue prefix: %w", err) + } + } + } + + // Project identity: the DB is authoritative when it already carries one; otherwise + // adopt the metadata.json id (a re-init or committed clone) or mint a fresh UUID. + // Write it to BOTH the DB and metadata.json so the two can never diverge (GH#2372) β€” + // the Dolt path keeps them in lockstep the same way. + projectID, _ := store.GetMetadata(ctx, "_project_id") + if projectID == "" { + if cfg.ProjectID != "" { + projectID = cfg.ProjectID + } else { + projectID = configfile.GenerateProjectID() + } + if err := store.SetMetadata(ctx, "_project_id", projectID); err != nil { + return fmt.Errorf("failed to write project ID: %w", err) + } + } + + // Persist metadata.json: backend + password-free DSN + schema + the DB's identity. + cfg.Backend = configfile.BackendPostgres + cfg.PostgresDSN = persistDSN + cfg.PostgresSchema = schema + cfg.ProjectID = projectID + if err := cfg.Save(in.beadsDir); err != nil { + return fmt.Errorf("failed to write metadata.json: %w", err) + } + + switch { + case in.jsonOutput: + out := map[string]any{ + "status": "initialized", + "backend": configfile.BackendPostgres, + "schema": schema, + "prefix": in.prefix, + } + b, _ := json.MarshalIndent(out, "", " ") + fmt.Println(string(b)) + case !in.quiet: + fmt.Printf("%s bd initialized with the Postgres backend\n", ui.RenderPass("βœ“")) + fmt.Printf(" Schema: %s\n", schema) + if in.prefix != "" { + fmt.Printf(" Prefix: %s\n", in.prefix) + } + fmt.Println(" History: not tracked (Postgres backend has no version control; use Dolt for history)") + } + return nil +} + +// initMySQLInput carries the resolved inputs the MySQL init arm needs. +type initMySQLInput struct { + beadsDir string + prefix string + mysqlURL string + mysqlDatabase string + quiet bool + jsonOutput bool +} + +// runInitMySQL finalizes a MySQL-backed workspace: it creates the per-workspace +// database, applies the schema (config seeds on first provision), seeds the issue +// prefix and project identity, and writes metadata.json with the password-stripped +// DSN. The password is supplied at open time via BEADS_MYSQL_PASSWORD and never lands +// on disk. MySQL isolates by database where Postgres isolates by schema. +func runInitMySQL(ctx context.Context, in initMySQLInput) error { + dsn := in.mysqlURL + if dsn == "" { + dsn = os.Getenv("BEADS_MYSQL_URL") + } + if dsn == "" { + return fmt.Errorf("bd init --backend=mysql requires --mysql-url= (or BEADS_MYSQL_URL), e.g. user:pass@tcp(host:3306)/ ; a password may be included for init but is never persisted β€” set BEADS_MYSQL_PASSWORD for later commands") + } + database := in.mysqlDatabase + if database == "" { + return fmt.Errorf("bd init --backend=mysql requires --mysql-database= (the per-workspace database)") + } + + // Redact BEFORE provisioning so a DSN we cannot safely persist fails before any + // side effect, and the password is guaranteed never to reach metadata.json. + persistDSN, err := beadsmysql.RedactPassword(dsn) + if err != nil { + return fmt.Errorf("cannot safely persist the MySQL connection: %w", err) + } + + store, err := beadsmysql.Provision(ctx, dsn, database) + if err != nil { + return fmt.Errorf("failed to initialize MySQL workspace: %w", err) + } + defer func() { _ = store.Close() }() + + cfg, _ := configfile.Load(in.beadsDir) + if cfg == nil { + cfg = configfile.DefaultConfig() + } + + if in.prefix != "" { + if existing, _ := store.GetConfig(ctx, "issue_prefix"); existing == "" { + issuePrefix := strings.ReplaceAll(in.prefix, ".", "_") + if err := store.SetConfig(ctx, "issue_prefix", issuePrefix); err != nil { + return fmt.Errorf("failed to set issue prefix: %w", err) + } + } + } + + projectID, _ := store.GetMetadata(ctx, "_project_id") + if projectID == "" { + if cfg.ProjectID != "" { + projectID = cfg.ProjectID + } else { + projectID = configfile.GenerateProjectID() + } + if err := store.SetMetadata(ctx, "_project_id", projectID); err != nil { + return fmt.Errorf("failed to write project ID: %w", err) + } + } + + cfg.Backend = configfile.BackendMySQL + cfg.MySQLDSN = persistDSN + cfg.MySQLDatabase = database + cfg.ProjectID = projectID + if err := cfg.Save(in.beadsDir); err != nil { + return fmt.Errorf("failed to write metadata.json: %w", err) + } + + switch { + case in.jsonOutput: + out := map[string]any{ + "status": "initialized", + "backend": configfile.BackendMySQL, + "database": database, + "prefix": in.prefix, + } + b, _ := json.MarshalIndent(out, "", " ") + fmt.Println(string(b)) + case !in.quiet: + fmt.Printf("%s bd initialized with the MySQL backend\n", ui.RenderPass("βœ“")) + fmt.Printf(" Database: %s\n", database) + if in.prefix != "" { + fmt.Printf(" Prefix: %s\n", in.prefix) + } + fmt.Println(" History: not tracked (MySQL backend has no version control; use Dolt for history)") + } + return nil +} + // initSQLiteInput carries the resolved inputs the SQLite init arm needs. type initSQLiteInput struct { beadsDir string @@ -1897,17 +2133,13 @@ func init() { initCmd.Flags().Bool("non-interactive", false, "Skip all interactive prompts (auto-detected in CI or non-TTY environments)") initCmd.Flags().String("role", "", "Set beads role without prompting: \"maintainer\" or \"contributor\"") - // Backend selection: Dolt (default) or embedded SQLite. - initCmd.Flags().String("backend", "", "Storage backend: dolt (default) or sqlite. See docs/architecture/storage-backends.md.") + // Backend selection: dolt (default) plus the SQL-family backends (postgres/mysql/sqlite). + initCmd.Flags().String("backend", "", "Storage backend: dolt (default), postgres, mysql, or sqlite. See docs/architecture/storage-backends.md.") + initCmd.Flags().String("pg-url", "", "Postgres connection URL (with --backend=postgres). A password may be included for init but is never persisted; set BEADS_PG_PASSWORD for later commands. Falls back to BEADS_POSTGRES_URL.") + initCmd.Flags().String("pg-schema", "", "Postgres schema for this workspace's tables (with --backend=postgres; provides search_path isolation)") + initCmd.Flags().String("mysql-url", "", "MySQL server DSN (with --backend=mysql), e.g. user:pass@tcp(host:3306)/ . A password may be included for init but is never persisted; set BEADS_MYSQL_PASSWORD for later commands. Falls back to BEADS_MYSQL_URL.") + initCmd.Flags().String("mysql-database", "", "MySQL database for this workspace (with --backend=mysql; MySQL's isolation unit)") initCmd.Flags().String("sqlite-path", "", "SQLite database file (with --backend=sqlite; relative to the beads dir, default beads.db)") - // Keep the short-lived server-backend flags as hidden parser tombstones. A - // legacy invocation can then reach the backend rollback guidance instead of - // failing early with an opaque "unknown flag" error. Current backends reject - // these flags explicitly above; they are never consumed as connection data. - for _, legacyFlag := range removedServerBackendInitFlags { - initCmd.Flags().String(legacyFlag.name, "", legacyFlag.usage) - _ = initCmd.Flags().MarkHidden(legacyFlag.name) - } // Dolt server connection flags initCmd.Flags().Bool("server", false, "Use external dolt sql-server instead of embedded engine") @@ -1937,16 +2169,6 @@ func init() { rootCmd.AddCommand(initCmd) } -var removedServerBackendInitFlags = []struct { - name string - usage string -}{ - {name: "pg-url", usage: "Legacy PostgreSQL connection URL (removed backend compatibility only)"}, - {name: "pg-schema", usage: "Legacy PostgreSQL schema (removed backend compatibility only)"}, - {name: "mysql-url", usage: "Legacy MySQL connection URL (removed backend compatibility only)"}, - {name: "mysql-database", usage: "Legacy MySQL database (removed backend compatibility only)"}, -} - // migrateOldDatabases detects and migrates old database files to beads.db func migrateOldDatabases(targetPath string, quiet bool) error { targetDir := filepath.Dir(targetPath) @@ -2035,14 +2257,36 @@ func alreadyInitialized(format string, args ...any) error { // A returned error that matches errWorkspaceAlreadyInitialized means a database // already exists (the benign, idempotent-skip case); any other error is // operational and must not be treated as success. -// checkExistingSQLBackend returns an "already initialized" guard for SQLite and a -// fail-loud tombstone error for workspaces created with a removed backend. These -// backends are marked by metadata.json alone, with no Dolt directory to inspect. +// checkExistingSQLBackend returns an "already initialized" guard error when cfg +// selects a non-Dolt SQL backend (Postgres/MySQL/SQLite), each of which is marked +// by metadata.json alone with no local DB directory to inspect. It returns nil for +// the Dolt backend so the caller runs the directory/server checks instead. Split +// out of checkExistingBeadsDataAt so the added SQL-backend branches don't inflate +// that function's complexity. func checkExistingSQLBackend(cfg *configfile.Config) error { - if err := validateConfiguredBackend(cfg); err != nil { - return err - } switch cfg.GetBackend() { + case configfile.BackendPostgres: + return alreadyInitialized(` +%s This workspace is already initialized with the Postgres backend (schema %q). + +To use it: + Just run bd commands normally (e.g., %s) + +To re-initialize (Postgres provisioning is idempotent): + bd init --backend=postgres --reinit-local ... + +Aborting.`, ui.RenderWarn("⚠"), cfg.GetPostgresSchema(), ui.RenderAccent("bd list")) + case configfile.BackendMySQL: + return alreadyInitialized(` +%s This workspace is already initialized with the MySQL backend (database %q). + +To use it: + Just run bd commands normally (e.g., %s) + +To re-initialize (MySQL provisioning is idempotent): + bd init --backend=mysql --reinit-local ... + +Aborting.`, ui.RenderWarn("⚠"), cfg.GetMySQLDatabase(), ui.RenderAccent("bd list")) case configfile.BackendSQLite: return alreadyInitialized(` %s This workspace is already initialized with the SQLite backend (%q). @@ -2065,24 +2309,20 @@ func checkExistingBeadsDataAt(beadsDir string, prefix string) error { } // metadata.json is authoritative for the configured backend, so resolve it once - // and dispatch. SQLite and removed-backend tombstones are marked by metadata - // alone β€” there is no local Dolt directory to inspect β€” so a plain + // and dispatch. A non-Dolt SQL backend (Postgres/MySQL/SQLite) is marked by + // metadata alone β€” there is no local DB directory to inspect β€” so a plain // `bd init` (which defaults to Dolt) must not silently repoint a live SQL // workspace to a fresh embedded Dolt DB and orphan its issues. --reinit-local - // /--force bypass this (handled by the caller). Invalid metadata must fail closed: - // without an explicit reinitialization request, init may not overwrite the only - // marker for an external or otherwise nonlocal database. + // /--force bypass this (handled by the caller). A config that fails to load falls + // through to the redirect/database-file checks below. cfg, cfgErr := configfile.Load(beadsDir) - if cfgErr != nil { - return fmt.Errorf("failed to load %s: %w; refusing to reinitialize automatically (restore the metadata or use --reinit-local after safeguarding existing data)", configfile.ConfigPath(beadsDir), cfgErr) - } - if cfg != nil { + if cfgErr == nil && cfg != nil { if guardErr := checkExistingSQLBackend(cfg); guardErr != nil { return guardErr } } - if cfg != nil && cfg.GetBackend() == configfile.BackendDolt { + if cfgErr == nil && cfg != nil && cfg.GetBackend() == configfile.BackendDolt { if cfg.IsDoltProxiedServerMode() { proxiedRoot, rootErr := resolveProxiedServerRootPath(beadsDir) if rootErr != nil { diff --git a/cmd/bd/init_embedded_test.go b/cmd/bd/init_embedded_test.go index bef8b52c2..409237153 100644 --- a/cmd/bd/init_embedded_test.go +++ b/cmd/bd/init_embedded_test.go @@ -1387,11 +1387,14 @@ func TestEmbeddedInit(t *testing.T) { t.Errorf("backend: got %q, want %q", cfg.Backend, configfile.BackendSQLite) } - for _, backend := range []string{"postgres", "mysql"} { - out := bdInitFail(t, bd, "--backend", backend) - if !strings.Contains(out, "no longer supported") { - t.Errorf("%s should report the backend rollback: %s", backend, out) - } + // Postgres is recognized, not an unknown backend: it fails only because the + // required connection config is absent (bdEnv strips any ambient DSN). + pgOut := bdInitFail(t, bd, "--backend", "postgres") + if strings.Contains(pgOut, "unknown backend") { + t.Errorf("postgres should be recognized, got unknown-backend error: %s", pgOut) + } + if !strings.Contains(pgOut, "--pg-url") { + t.Errorf("postgres init should request --pg-url, got: %s", pgOut) } // A genuinely unsupported backend is still rejected. diff --git a/cmd/bd/init_test.go b/cmd/bd/init_test.go index 2ddd728a7..c152c9a13 100644 --- a/cmd/bd/init_test.go +++ b/cmd/bd/init_test.go @@ -2396,28 +2396,28 @@ func TestInitBackendFlag(t *testing.T) { } }) - for _, backend := range []string{"postgres", "mysql"} { - t.Run(backend+"_is_no_longer_supported", func(t *testing.T) { - tmpDir := t.TempDir() - beadsDir := filepath.Join(tmpDir, ".beads") + // Postgres is a recognized backend now: it must not be rejected as unknown, + // and it fails only because the required connection config is absent. + t.Run("postgres_is_recognized_and_requires_config", func(t *testing.T) { + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") - cmd := exec.Command(bd, "init", "--backend", backend, "--quiet") - cmd.Dir = tmpDir - cmd.Env = initBackendTestEnv(beadsDir) - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("Expected non-zero exit for --backend=%s", backend) - } + cmd := exec.Command(bd, "init", "--backend", "postgres", "--quiet") + cmd.Dir = tmpDir + cmd.Env = initBackendTestEnv(beadsDir) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatal("Expected non-zero exit for --backend=postgres without connection config") + } - outStr := string(out) - if !strings.Contains(outStr, "no longer supported") { - t.Errorf("Expected rollback guidance for %s, got: %s", backend, outStr) - } - if !strings.Contains(outStr, `"dolt"`) || !strings.Contains(outStr, `"sqlite"`) { - t.Errorf("Expected supported backend guidance for %s, got: %s", backend, outStr) - } - }) - } + outStr := string(out) + if strings.Contains(outStr, "unknown backend") { + t.Errorf("postgres must be a recognized backend, not reported unknown: %s", outStr) + } + if !strings.Contains(outStr, "--pg-url") { + t.Errorf("Expected missing --pg-url guidance for postgres, got: %s", outStr) + } + }) // A genuinely unsupported backend value is still rejected up front. t.Run("unknown_backend_errors", func(t *testing.T) { diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 0e1bb6765..3929f8e51 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -33,6 +33,9 @@ import ( "github.com/steveyegge/beads/internal/routing" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dolt" + mysqlstore "github.com/steveyegge/beads/internal/storage/mysql" + "github.com/steveyegge/beads/internal/storage/pgdialect" + pgstore "github.com/steveyegge/beads/internal/storage/postgres" "github.com/steveyegge/beads/internal/storage/schema" sqlitestore "github.com/steveyegge/beads/internal/storage/sqlite" "github.com/steveyegge/beads/internal/storage/uow" @@ -989,13 +992,9 @@ var rootCmd = &cobra.Command{ if dbPath == "" { if bd := beads.FindBeadsDir(); bd != "" { - cfg, cfgErr := configfile.Load(bd) - if cfgErr != nil || cfg != nil && (cfg.IsDoltProxiedServerMode() || cfg.GetBackend() == configfile.BackendSQLite || !configfile.IsSupportedBackend(cfg.Backend)) { - // SQLite, proxied-server, and removed-backend workspaces may have no - // local Dolt database file. Invalid or unknown metadata likewise must - // reach config validation instead of becoming a generic "no database" - // result. metadata.json identifies the workspace so store selection can - // route or reject it explicitly. + if cfg, _ := configfile.Load(bd); cfg != nil && (cfg.IsDoltProxiedServerMode() || cfg.GetBackend() == configfile.BackendPostgres || cfg.GetBackend() == configfile.BackendMySQL || cfg.GetBackend() == configfile.BackendSQLite) { + // A non-Dolt SQL (or proxied-server) workspace has no local Dolt + // database file; the .beads dir with metadata.json IS the workspace. dbPath = bd } } @@ -1077,18 +1076,6 @@ var rootCmd = &cobra.Command{ return HandleError("%v", err) } - // Resolve the backend before version tracking, migration, server startup, or - // any store construction. PostgreSQL/MySQL values are retained as metadata - // tombstones so an existing workspace fails closed instead of falling through - // to a new, empty Dolt database. - cfg, cfgErr := configfile.Load(beadsDir) - if cfgErr != nil { - return HandleError("failed to load beads config from %s: %v (refusing to fall back to the embedded store; fix or restore metadata.json and retry)", beadsDir, cfgErr) - } - if backendErr := validateConfiguredBackend(cfg); backendErr != nil { - return HandleError("%v", backendErr) - } - // Set actor for audit trail actor = getActorWithGit() // Attach actor to the command span now that we have it. @@ -1147,6 +1134,10 @@ var rootCmd = &cobra.Command{ // result set and exit 0 (false-empty), which readers misinterpret as // "no work". Absent metadata.json (cfg == nil, cfgErr == nil) keeps // the fresh-repo embedded default below. + cfg, cfgErr := configfile.Load(beadsDir) + if cfgErr != nil { + return HandleError("failed to load beads config from %s: %v (refusing to fall back to the embedded store; fix or restore metadata.json and retry)", beadsDir, cfgErr) + } if cfg != nil { warnSharedServerEmbeddedMismatch(cfg) doltCfg.ProxiedServer = cfg.IsDoltProxiedServerMode() @@ -1188,7 +1179,9 @@ var rootCmd = &cobra.Command{ // command must not run (or fail) embedded opens even when the env var is set. // Dolt-only: the gateway credential command mints a Dolt server // username. IsSharedServerMode() forces ServerMode true with no backend - // guard, so SQLite must not try to resolve a server username. + // guard, so without this check a shared-server config in config.yaml + // would run (and fail closed) the command for postgres/mysql/sqlite + // workspaces, which never present a server username. if doltCfg.ServerMode && cfg.GetBackend() == configfile.BackendDolt { if _, credErr := dolt.ApplyGatewayCredential(rootCtx, cfg, doltCfg); credErr != nil { return HandleError("resolving dolt credential command: %v", credErr) @@ -1282,7 +1275,14 @@ var rootCmd = &cobra.Command{ // Removing them WILL cause unrecoverable data corruption and data loss. // Dolt manages these files itself; external interference is never safe. - if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { + if cfg != nil && cfg.GetBackend() == configfile.BackendPostgres { + // Postgres backend: open via the SQL-family bundle, bypassing the + // Dolt open path (the doltCfg above is built but unused here). + store, err = pgstore.NewFromConfig(rootCtx, beadsDir) + } else if cfg != nil && cfg.GetBackend() == configfile.BackendMySQL { + // MySQL backend: same SQL-family bundle, isolation by database. + store, err = mysqlstore.NewFromConfig(rootCtx, beadsDir) + } else if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { // SQLite backend: pure-Go file-based SQL-family bundle. store, err = sqlitestore.NewFromConfig(rootCtx, beadsDir) } else { @@ -1394,7 +1394,7 @@ var rootCmd = &cobra.Command{ uowProvider = nil } } else { - // A NonCommitGraphBackend such as SQLite has no Dolt + // Slice 5c: a NonCommitGraphBackend (Postgres, SQLite, ...) has no Dolt // commit graph, so skip the Dolt-only maintenance tail. UnwrapStore reaches // the concrete store past the HookFiringStore decorator; a store WITHOUT the // marker (every Dolt variant) leaves skipMaintenance false and runs the tail @@ -1572,7 +1572,7 @@ func checkBlockedEnvVars() error { for _, name := range blockedEnvVars { if os.Getenv(name) != "" { return fmt.Errorf("%s env var is not supported and has been removed to prevent data fragmentation.\n"+ - "Unset %s; storage selection comes from .beads/metadata.json. To choose a different supported backend, follow 'bd help init-safety'; do not edit metadata.json by hand", name, name) + "The storage backend is set in .beads/metadata.json. To change it, use: bd migrate dolt", name) } } return nil @@ -1747,9 +1747,15 @@ func envTruthyValue(v string) bool { return true } +// dsnFlags are the flags whose value is a connection string that may embed a +// password (bd init --backend=postgres/mysql). Their values get the full +// parser-backed DSN scrub; other args only get the unambiguous userinfo scrub. +var dsnFlags = map[string]bool{"--pg-url": true, "--mysql-url": true} + // secretFlagNames are long flag names whose entire value is an opaque credential -// that must never reach the bd.args telemetry span. The flag's value is redacted -// wholesale. Only federation add-peer's --password currently qualifies. Its shorthand (-p) is +// that must never reach the bd.args telemetry span. Unlike dsnFlags (a DSN that is +// parsed so its structure survives), a secret flag's value is redacted wholesale. +// Only federation add-peer's --password currently qualifies. Its shorthand (-p) is // resolved per command via secretFlagTokens so the same letter bound to // --priority/--prefix/--parallel on other commands is never redacted. var secretFlagNames = map[string]bool{"password": true} @@ -1778,38 +1784,40 @@ func secretFlagTokens(cmd *cobra.Command) map[string]bool { } // scrubArgsForTelemetry joins argv for the bd.args span attribute with any -// credential-bearing values redacted. A secretFlags token's value is redacted -// wholesale across the `--password `, +// credential-bearing values redacted. --pg-url/--mysql-url may carry a password +// for init, and federation add-peer --password/-p carries a SQL password; +// RedactPassword protects metadata.json, but the raw argv would otherwise leak the +// secret into the telemetry root span and trace logs. +// +// A DSN flag's value is scrubbed through the parser-backed pgdialect logic so every +// password form pgx accepts is caught β€” URL userinfo, URL `?password=`/`?sslpassword=` +// query params, and libpq `password=`/`sslpassword=` keyword/value tokens β€” in both +// the `--pg-url=` and `--pg-url ` spellings. A secretFlags token's value is +// an opaque credential and is redacted wholesale across the `--password `, // `--password=`, `-p `, `-p=`, and `-p` spellings pflag accepts. Every -// other arg gets a conservative DSN/userinfo scrub as defense in depth so a -// positional connection string cannot leak a password. +// other arg still gets the narrow user:PASS@host userinfo scrub as defense in depth, +// without the broad keyword scan that would over-redact ordinary text. func scrubArgsForTelemetry(argv []string, secretFlags map[string]bool) string { parts := make([]string, len(argv)) - redactNext := false for i, a := range argv { - if redactNext { - parts[i] = "xxxxx" - redactNext = false - continue - } - if name, value, ok := strings.Cut(a, "="); ok { + if name, val, ok := strings.Cut(a, "="); ok { + if dsnFlags[name] { + // --pg-url= β€” scrub only the value, keep the flag name. + parts[i] = name + "=" + scrubDSNValue(val) + continue + } if secretFlags[name] { // --password= / -p= β€” redact the whole value. parts[i] = name + "=xxxxx" continue } - if strings.HasPrefix(name, "-") { - scrubbed := scrubUserinfoPassword(scrubPotentialDSNPasswords(value)) - if scrubbed != value { - // Preserve an arbitrary flag name while parsing its equals-value as - // a possible DSN. Passing the whole token to url.Parse would treat - // the flag prefix as the URL scheme and miss query credentials. - parts[i] = name + "=" + scrubbed - continue - } - } } if i > 0 { + if dsnFlags[argv[i-1]] { + // following a bare --pg-url token. + parts[i] = scrubDSNValue(a) + continue + } if secretFlags[argv[i-1]] { // following a bare --password / -p token. parts[i] = "xxxxx" @@ -1821,14 +1829,7 @@ func scrubArgsForTelemetry(argv []string, secretFlags map[string]bool) string { parts[i] = short + "xxxxx" continue } - if secretShorthandTakesSeparateValue(a, secretFlags) { - // -qp β€” a boolean shorthand cluster ending in the - // value-taking secret shorthand, with its value in the next token. - parts[i] = a - redactNext = true - continue - } - parts[i] = scrubUserinfoPassword(scrubPotentialDSNPasswords(a)) + parts[i] = scrubUserinfoPassword(a) } return strings.Join(parts, " ") } @@ -1867,23 +1868,12 @@ func secretShorthandPrefix(a string, secretFlags map[string]bool) (string, bool) return "", false } -// secretShorthandTakesSeparateValue recognizes a boolean-shorthand cluster that -// ends in a registered secret shorthand with no attached value. For example, -// pflag parses "-qp secret" as -q followed by -p=secret. -func secretShorthandTakesSeparateValue(a string, secretFlags map[string]bool) bool { - if len(a) < 3 || a[0] != '-' || a[1] == '-' { - return false - } - for i := 1; i < len(a); i++ { - c := a[i] - if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { - return false - } - if secretFlags["-"+string(c)] { - return i == len(a)-1 - } - } - return false +// scrubDSNValue redacts every password form from a connection-string value. The +// pgdialect pass covers pgx's URL (userinfo + query) and libpq keyword/value shapes; +// the trailing userinfo pass covers the MySQL user:PASS@tcp(...) DSN, which is not a +// scheme URL and so is invisible to the pgdialect extractor. +func scrubDSNValue(val string) string { + return scrubUserinfoPassword(pgdialect.ScrubDSNString(val, val)) } // scrubUserinfoPassword redacts the password in a URL/DSN userinfo section diff --git a/cmd/bd/main_test.go b/cmd/bd/main_test.go index 187cc130f..6563ed78c 100644 --- a/cmd/bd/main_test.go +++ b/cmd/bd/main_test.go @@ -142,12 +142,6 @@ func TestBlockedEnvVars(t *testing.T) { if err != nil && !strings.Contains(err.Error(), tt.envVar) { t.Errorf("expected error to mention %s, got: %v", tt.envVar, err) } - if err != nil && !strings.Contains(err.Error(), "bd help init-safety") { - t.Errorf("expected error to point to safe reinitialization guidance, got: %v", err) - } - if err != nil && strings.Contains(err.Error(), "bd migrate dolt") { - t.Errorf("error must not recommend the removed backend-conversion command: %v", err) - } }) } diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index 283c375a7..6fdac6657 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -15,6 +15,8 @@ import ( "github.com/steveyegge/beads/internal/storage/dbproxy/util" "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/storage/embeddeddolt" + beadsmysql "github.com/steveyegge/beads/internal/storage/mysql" + "github.com/steveyegge/beads/internal/storage/postgres" beadssqlite "github.com/steveyegge/beads/internal/storage/sqlite" ) @@ -108,8 +110,11 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltS // metadata.json (cfg == nil, err == nil) keeps the embedded default. return nil, fmt.Errorf("load %s: %w (refusing to fall back to the embedded store)", configfile.ConfigPath(beadsDir), err) } - if err := validateConfiguredBackend(cfg); err != nil { - return nil, err + if cfg != nil && cfg.GetBackend() == configfile.BackendPostgres { + return postgres.NewFromConfig(ctx, beadsDir) + } + if cfg != nil && cfg.GetBackend() == configfile.BackendMySQL { + return beadsmysql.NewFromConfig(ctx, beadsDir) } if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { return beadssqlite.NewFromConfig(ctx, beadsDir) @@ -194,8 +199,14 @@ func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.D // "database not found" the embedded open would produce. return nil, fmt.Errorf("load %s: %w (refusing to fall back to the embedded store)", configfile.ConfigPath(beadsDir), err) } - if err := validateConfiguredBackend(cfg); err != nil { - return nil, err + if cfg != nil && cfg.GetBackend() == configfile.BackendPostgres { + // Postgres has no read-only open mode in the wedge; a normal open is fine + // (reads don't mutate, and search_path is per-workspace). + return postgres.NewFromConfig(ctx, beadsDir) + } + if cfg != nil && cfg.GetBackend() == configfile.BackendMySQL { + // MySQL likewise has no separate read-only open in the wedge; reads don't mutate. + return beadsmysql.NewFromConfig(ctx, beadsDir) } if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { return beadssqlite.NewFromConfig(ctx, beadsDir) diff --git a/cmd/bd/store_factory_nocgo.go b/cmd/bd/store_factory_nocgo.go index d66a380aa..892eb0f0f 100644 --- a/cmd/bd/store_factory_nocgo.go +++ b/cmd/bd/store_factory_nocgo.go @@ -10,6 +10,8 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dbproxy/util" "github.com/steveyegge/beads/internal/storage/dolt" + beadsmysql "github.com/steveyegge/beads/internal/storage/mysql" + "github.com/steveyegge/beads/internal/storage/postgres" beadssqlite "github.com/steveyegge/beads/internal/storage/sqlite" ) @@ -55,8 +57,13 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltS // message below. return nil, fmt.Errorf("load %s: %w", configfile.ConfigPath(beadsDir), err) } - if err := validateConfiguredBackend(cfg); err != nil { - return nil, err + if cfg != nil && cfg.GetBackend() == configfile.BackendPostgres { + // Postgres needs no CGO (pure-Go pgx), so it works in the nocgo build too. + return postgres.NewFromConfig(ctx, beadsDir) + } + if cfg != nil && cfg.GetBackend() == configfile.BackendMySQL { + // MySQL (go-sql-driver) needs no CGO either. + return beadsmysql.NewFromConfig(ctx, beadsDir) } if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { // SQLite (modernc.org/sqlite) is pure-Go; no CGO. @@ -83,8 +90,11 @@ func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.D if err != nil { return nil, fmt.Errorf("load %s: %w", configfile.ConfigPath(beadsDir), err) } - if err := validateConfiguredBackend(cfg); err != nil { - return nil, err + if cfg != nil && cfg.GetBackend() == configfile.BackendPostgres { + return postgres.NewFromConfig(ctx, beadsDir) + } + if cfg != nil && cfg.GetBackend() == configfile.BackendMySQL { + return beadsmysql.NewFromConfig(ctx, beadsDir) } if cfg != nil && cfg.GetBackend() == configfile.BackendSQLite { return beadssqlite.NewFromConfig(ctx, beadsDir) diff --git a/cmd/bd/store_selection_removed_backend_test.go b/cmd/bd/store_selection_removed_backend_test.go deleted file mode 100644 index 66bd224f1..000000000 --- a/cmd/bd/store_selection_removed_backend_test.go +++ /dev/null @@ -1,403 +0,0 @@ -package main - -import ( - "context" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - "github.com/steveyegge/beads/internal/configfile" -) - -func TestCommandStartupRejectsRemovedBackendsBeforeLocalWrites(t *testing.T) { - bd := buildBDForInitTests(t) - for _, backend := range []string{configfile.BackendPostgres, configfile.BackendMySQL} { - t.Run(backend, func(t *testing.T) { - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(beadsDir, 0o755); err != nil { - t.Fatalf("create beads dir: %v", err) - } - if err := (&configfile.Config{Backend: backend}).Save(beadsDir); err != nil { - t.Fatalf("save config: %v", err) - } - - cmd := exec.Command(bd, "list", "--json") - cmd.Dir = root - cmd.Env = append(removedBackendTestEnv(beadsDir), "BD_DISABLE_METRICS=1") - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("removed backend %s unexpectedly opened", backend) - } - message := strings.ToLower(string(out)) - for _, want := range []string{"no longer supported", "simple and resource-light", "was not opened", "export", "dolt", "sqlite"} { - if !strings.Contains(message, want) { - t.Errorf("startup error for %s missing %q: %s", backend, want, message) - } - } - - for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version"} { - path := filepath.Join(beadsDir, name) - if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { - t.Fatalf("removed backend %s created local state at %s (stat error: %v)", backend, path, statErr) - } - } - }) - } -} - -func TestCommandStartupRejectsUnknownBackendBeforeLocalWrites(t *testing.T) { - bd := buildBDForInitTests(t) - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(beadsDir, 0o700); err != nil { - t.Fatalf("create .beads: %v", err) - } - if err := (&configfile.Config{Backend: "mystery"}).Save(beadsDir); err != nil { - t.Fatalf("save unknown-backend metadata: %v", err) - } - - cmd := exec.Command(bd, "list", "--json") - cmd.Dir = root - cmd.Env = append(removedBackendTestEnv(beadsDir), "BD_DISABLE_METRICS=1") - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("unknown backend unexpectedly opened:\n%s", out) - } - message := string(out) - if strings.Contains(message, "no beads database found") { - t.Fatalf("unknown backend stopped at generic discovery instead of metadata guidance:\n%s", out) - } - for _, want := range []string{"not recognized", "opened or modified", "dolt", "sqlite"} { - if !strings.Contains(message, want) { - t.Errorf("unknown-backend error missing %q:\n%s", want, out) - } - } - for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version"} { - if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { - t.Fatalf("unknown backend created %s (stat error: %v)", name, statErr) - } - } -} - -func TestInitGuardRejectsCorruptMetadataBeforeWorkspaceWrites(t *testing.T) { - beadsDir := filepath.Join(t.TempDir(), ".beads") - if err := os.MkdirAll(beadsDir, 0o700); err != nil { - t.Fatalf("create .beads: %v", err) - } - metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) - metadata := []byte("{\n") - if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { - t.Fatalf("write corrupt metadata.json: %v", err) - } - - err := checkExistingBeadsDataAt(beadsDir, "guard") - if err == nil || !strings.Contains(err.Error(), "metadata.json") { - t.Fatalf("init guard error = %v, want corrupt metadata refusal", err) - } - after, readErr := os.ReadFile(metadataPath) - if readErr != nil { - t.Fatalf("read metadata after refusal: %v", readErr) - } - if string(after) != string(metadata) { - t.Fatalf("init guard rewrote corrupt metadata: before %q, after %q", metadata, after) - } - for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version"} { - if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { - t.Fatalf("init guard created %s (stat error: %v)", name, statErr) - } - } -} - -func TestDoctorRejectsUnknownBackendBeforeDoltChecks(t *testing.T) { - bd := buildBDForInitTests(t) - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(beadsDir, 0o700); err != nil { - t.Fatalf("create .beads: %v", err) - } - cfg := &configfile.Config{ - Backend: "mystery", - DoltMode: configfile.DoltModeServer, - DoltServerHost: "127.0.0.1", - DoltServerPort: 1, - } - if err := cfg.Save(beadsDir); err != nil { - t.Fatalf("save unknown-backend metadata: %v", err) - } - - cmd := exec.Command(bd, "doctor", "--server", "--json") - cmd.Dir = root - cmd.Env = append(removedBackendTestEnv(beadsDir), "BEADS_DOLT_AUTO_START=0", "BD_DISABLE_METRICS=1") - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("doctor unexpectedly ran Dolt checks for unknown backend:\n%s", out) - } - for _, want := range []string{"not recognized", "no storage database was opened or modified", "dolt", "sqlite"} { - if !strings.Contains(string(out), want) { - t.Errorf("doctor unknown-backend error missing %q:\n%s", want, out) - } - } - for _, name := range []string{"embeddeddolt", "dolt", "beads.db", "dolt-server.pid", "dolt-server.port"} { - if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { - t.Fatalf("doctor created %s for unknown backend (stat error: %v)", name, statErr) - } - } -} - -func TestLegacyInitFlagsReachRemovedBackendGuidance(t *testing.T) { - bd := buildBDForInitTests(t) - tests := []struct { - backend string - args []string - }{ - { - backend: configfile.BackendPostgres, - args: []string{ - "init", "--backend=postgres", - "--pg-url=postgres://beads@127.0.0.1:5432/beads", - "--pg-schema=legacy_workspace", - }, - }, - { - backend: configfile.BackendMySQL, - args: []string{ - "init", "--backend=mysql", - "--mysql-url=beads@tcp(127.0.0.1:3306)/", - "--mysql-database=legacy_workspace", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.backend, func(t *testing.T) { - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - cmd := exec.Command(bd, tt.args...) - cmd.Dir = root - cmd.Env = append(removedBackendTestEnv(beadsDir), "BD_DISABLE_METRICS=1") - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("legacy %s init unexpectedly succeeded: %s", tt.backend, out) - } - - message := string(out) - if strings.Contains(message, "unknown flag") { - t.Errorf("legacy %s init stopped at argument parsing instead of rollback guidance: %s", tt.backend, message) - } - for _, want := range []string{"no longer supported", "general-purpose server databases", "simple and resource-light", "dolt", "sqlite"} { - if !strings.Contains(message, want) { - t.Errorf("legacy %s init error missing %q: %s", tt.backend, want, message) - } - } - if _, statErr := os.Stat(beadsDir); !os.IsNotExist(statErr) { - t.Fatalf("rejected legacy %s init created workspace state (stat error: %v)", tt.backend, statErr) - } - }) - } -} - -func TestDoltAdministrativeCommandsRejectRemovedBackends(t *testing.T) { - bd := buildBDForInitTests(t) - commands := []struct { - name string - args []string - }{ - {name: "show", args: []string{"dolt", "show"}}, - {name: "status", args: []string{"dolt", "status"}}, - {name: "set", args: []string{"dolt", "set", "host", "127.0.0.1"}}, - {name: "test", args: []string{"dolt", "test"}}, - {name: "start", args: []string{"dolt", "start"}}, - {name: "stop", args: []string{"dolt", "stop"}}, - {name: "killall", args: []string{"dolt", "killall"}}, - {name: "clean-databases", args: []string{"dolt", "clean-databases", "--dry-run"}}, - } - - for _, backend := range []string{configfile.BackendPostgres, configfile.BackendMySQL} { - t.Run(backend, func(t *testing.T) { - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(beadsDir, 0o755); err != nil { - t.Fatalf("create beads dir: %v", err) - } - if err := (&configfile.Config{Backend: backend}).Save(beadsDir); err != nil { - t.Fatalf("save config: %v", err) - } - - for _, command := range commands { - t.Run(command.name, func(t *testing.T) { - cmd := exec.Command(bd, command.args...) - cmd.Dir = root - cmd.Env = append(removedBackendTestEnv(beadsDir), "BD_DISABLE_METRICS=1") - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("%q unexpectedly succeeded for removed backend %s: %s", strings.Join(command.args, " "), backend, out) - } - - message := strings.ToLower(string(out)) - for _, want := range []string{"no longer supported", "simple and resource-light", "was not opened", "export", "dolt", "sqlite"} { - if !strings.Contains(message, want) { - t.Errorf("%q error for %s missing %q: %s", strings.Join(command.args, " "), backend, want, message) - } - } - - for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version"} { - path := filepath.Join(beadsDir, name) - if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { - t.Fatalf("%q created local state for removed backend %s at %s (stat error: %v)", strings.Join(command.args, " "), backend, path, statErr) - } - } - }) - } - }) - } -} - -func TestDoltReadOnlyCommandsRejectUnknownBackend(t *testing.T) { - bd := buildBDForInitTests(t) - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(beadsDir, 0o700); err != nil { - t.Fatalf("create .beads: %v", err) - } - if err := (&configfile.Config{Backend: "mystery", DoltMode: configfile.DoltModeServer, DoltServerPort: 1}).Save(beadsDir); err != nil { - t.Fatalf("save unknown-backend metadata: %v", err) - } - - for _, args := range [][]string{{"dolt", "show"}, {"dolt", "status"}, {"dolt", "killall"}} { - cmd := exec.Command(bd, args...) - cmd.Dir = root - cmd.Env = append(removedBackendTestEnv(beadsDir), "BEADS_DOLT_AUTO_START=0", "BD_DISABLE_METRICS=1") - out, err := cmd.CombinedOutput() - if err == nil { - t.Errorf("bd %s unexpectedly treated unknown metadata as Dolt:\n%s", strings.Join(args, " "), out) - continue - } - if !strings.Contains(string(out), "not recognized") || !strings.Contains(string(out), "opened or modified") { - t.Errorf("bd %s error lacks fail-closed guidance:\n%s", strings.Join(args, " "), out) - } - } -} - -func TestDoltKillallRejectsCorruptMetadataBeforeServerCleanup(t *testing.T) { - bd := buildBDForInitTests(t) - root := t.TempDir() - beadsDir := filepath.Join(root, ".beads") - if err := os.MkdirAll(beadsDir, 0o700); err != nil { - t.Fatalf("create .beads: %v", err) - } - metadata := []byte("{\n") - metadataPath := filepath.Join(beadsDir, configfile.ConfigFileName) - if err := os.WriteFile(metadataPath, metadata, 0o600); err != nil { - t.Fatalf("write corrupt metadata: %v", err) - } - - cmd := exec.Command(bd, "dolt", "killall") - cmd.Dir = root - cmd.Env = append(removedBackendTestEnv(beadsDir), - "BEADS_DOLT_SHARED_SERVER=1", - "BEADS_DOLT_AUTO_START=0", - "BEADS_SHARED_SERVER_DIR="+filepath.Join(root, "shared-server"), - "BD_DISABLE_METRICS=1", - ) - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("bd dolt killall unexpectedly ignored corrupt metadata:\n%s", out) - } - message := strings.ToLower(string(out)) - for _, want := range []string{"loading config", "parsing config"} { - if !strings.Contains(message, want) { - t.Errorf("bd dolt killall corrupt-metadata error missing %q:\n%s", want, out) - } - } - after, readErr := os.ReadFile(metadataPath) - if readErr != nil { - t.Fatalf("read metadata after refusal: %v", readErr) - } - if string(after) != string(metadata) { - t.Fatalf("bd dolt killall rewrote corrupt metadata: before %q, after %q", metadata, after) - } - for _, name := range []string{"embeddeddolt", "dolt", "beads.db", ".local_version", "dolt-server.pid", "dolt-server.port"} { - if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { - t.Fatalf("bd dolt killall created %s before rejecting corrupt metadata (stat error: %v)", name, statErr) - } - } -} - -func removedBackendTestEnv(beadsDir string) []string { - var env []string - for _, entry := range os.Environ() { - if strings.HasPrefix(entry, "BEADS_") || strings.HasPrefix(entry, "BD_") { - continue - } - env = append(env, entry) - } - return append(env, "BEADS_DIR="+beadsDir) -} - -func TestStoreFactoriesRemovedBackendsFailLoud(t *testing.T) { - for _, backend := range []string{configfile.BackendPostgres, configfile.BackendMySQL} { - t.Run(backend, func(t *testing.T) { - beadsDir := t.TempDir() - cfg := &configfile.Config{Backend: backend} - if err := cfg.Save(beadsDir); err != nil { - t.Fatalf("save config: %v", err) - } - - factories := map[string]func(context.Context, string) (interface{ Close() error }, error){ - "read-write": func(ctx context.Context, dir string) (interface{ Close() error }, error) { - return newDoltStoreFromConfig(ctx, dir) - }, - "read-only": func(ctx context.Context, dir string) (interface{ Close() error }, error) { - return newReadOnlyStoreFromConfig(ctx, dir) - }, - } - - for name, open := range factories { - t.Run(name, func(t *testing.T) { - store, err := open(t.Context(), beadsDir) - if err == nil { - if store != nil { - _ = store.Close() - } - t.Fatalf("expected removed-backend error for %s", backend) - } - if !strings.Contains(err.Error(), "no longer supported") { - t.Fatalf("error should explain that %s is no longer supported: %v", backend, err) - } - guidance := strings.ToLower(err.Error()) - if !strings.Contains(guidance, "export") || !strings.Contains(guidance, "dolt") || !strings.Contains(guidance, "sqlite") { - t.Fatalf("error should provide safe migration guidance: %v", err) - } - for _, name := range []string{"embeddeddolt", "dolt", "beads.db"} { - path := filepath.Join(beadsDir, name) - if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { - t.Fatalf("removed backend %s created local storage at %s (stat error: %v)", backend, path, statErr) - } - } - }) - } - }) - } -} - -func TestRequireDoltBackend(t *testing.T) { - for _, cfg := range []*configfile.Config{nil, {}, {Backend: configfile.BackendDolt}} { - if err := requireDoltBackend(cfg); err != nil { - t.Fatalf("Dolt config %#v rejected: %v", cfg, err) - } - } - - if err := requireDoltBackend(&configfile.Config{Backend: configfile.BackendSQLite}); err == nil || !strings.Contains(err.Error(), "not using Dolt") { - t.Fatalf("SQLite Dolt-command guard error = %v, want non-Dolt explanation", err) - } - - for _, backend := range []string{configfile.BackendPostgres, configfile.BackendMySQL} { - err := requireDoltBackend(&configfile.Config{Backend: backend}) - if err == nil || !strings.Contains(err.Error(), "no longer supported") || !strings.Contains(err.Error(), "export") { - t.Fatalf("removed backend %q guard error = %v, want rollback and migration guidance", backend, err) - } - } -} diff --git a/cmd/bd/telemetry_redact.go b/cmd/bd/telemetry_redact.go deleted file mode 100644 index 0be0b2d1e..000000000 --- a/cmd/bd/telemetry_redact.go +++ /dev/null @@ -1,129 +0,0 @@ -package main - -import ( - "net/url" - "regexp" - "sort" - "strings" -) - -// These expressions recognize password-bearing libpq keyword/value tokens. The -// scrubber is intentionally backend-neutral: command-line telemetry may receive a -// connection string even when bd does not implement that connection's backend. -const dsnWhitespace = `\t\n\f\r \v` - -var dsnPasswordValue = regexp.MustCompile(`(?i)(?:^|[` + dsnWhitespace + `])(?:password|sslpassword)[` + dsnWhitespace + `]*=[` + dsnWhitespace + `]*(?:'((?:[^'\\]|\\.)*)'|((?:[^` + dsnWhitespace + `\\]|\\.)+))`) - -// A libpq keyword/value connection string begins with a recognized connection -// parameter. Requiring the first key to be recognized avoids treating ordinary -// prose such as "transport=rail password=..." as a DSN while still covering valid -// minimal forms such as "password=..." and service-file forms. -var dsnLeadingKeyword = regexp.MustCompile(`(?i)^[` + dsnWhitespace + `]*(?:application_name|channel_binding|client_encoding|connect_timeout|dbname|fallback_application_name|gssencmode|gsslib|host|hostaddr|keepalives|keepalives_count|keepalives_idle|keepalives_interval|krbsrvname|load_balance_hosts|options|passfile|password|port|replication|requirepeer|service|servicefile|sslcert|sslcrl|sslcrldir|sslkey|sslmode|sslpassword|sslrootcert|sslsni|ssl_max_protocol_version|ssl_min_protocol_version|target_session_attrs|tcp_user_timeout|user)[` + dsnWhitespace + `]*=`) - -func scrubStructuredDSNPasswords(dsn, text string) string { - secrets := structuredDSNPasswords(dsn) - sort.Slice(secrets, func(i, j int) bool { return len(secrets[i]) > len(secrets[j]) }) - for _, secret := range secrets { - text = strings.ReplaceAll(text, secret, "xxxxx") - } - return text -} - -func scrubPotentialDSNPasswords(value string) string { - if strings.Contains(value, "://") { - return scrubStructuredDSNPasswords(value, value) - } - if dsnLeadingKeyword.MatchString(value) { - return scrubStructuredDSNPasswords(value, value) - } - return value -} - -func structuredDSNPasswords(dsn string) []string { - seen := make(map[string]bool) - var values []string - add := func(value string) { - if value == "" || seen[value] { - return - } - seen[value] = true - values = append(values, value) - } - - if parsed, err := url.Parse(dsn); err == nil && parsed.Scheme != "" { - if parsed.User != nil { - if password, ok := parsed.User.Password(); ok { - add(password) - add(rawURLUserinfoPassword(dsn)) - } - } - } - // Extract query credentials independently of url.Parse. A malformed escape in - // the path, userinfo, or fragment makes url.Parse reject the entire URL even - // when the password query pair itself is valid. Telemetry redaction must fail - // closed in that case. - addRawURLQueryPasswords(dsn, add) - - for _, match := range dsnPasswordValue.FindAllStringSubmatch(dsn, -1) { - add(match[1]) - add(match[2]) - } - return values -} - -func addRawURLQueryPasswords(dsn string, add func(string)) { - queryStart := strings.IndexByte(dsn, '?') - if queryStart < 0 { - return - } - query := dsn[queryStart+1:] - if fragment := strings.IndexByte(query, '#'); fragment >= 0 { - query = query[:fragment] - } - for _, pair := range strings.Split(query, "&") { - key, value, ok := strings.Cut(pair, "=") - if !ok { - continue - } - decodedKey, err := url.QueryUnescape(key) - if err != nil { - decodedKey = key - } - if !isDSNPasswordKey(key) && !isDSNPasswordKey(decodedKey) { - continue - } - add(value) - if decoded, err := url.QueryUnescape(value); err == nil { - add(decoded) - } - } -} - -func isDSNPasswordKey(key string) bool { - switch strings.ToLower(key) { - case "password", "sslpassword": - return true - default: - return false - } -} - -func rawURLUserinfoPassword(dsn string) string { - scheme := strings.Index(dsn, "://") - if scheme < 0 { - return "" - } - authority := dsn[scheme+len("://"):] - if end := strings.IndexAny(authority, "/?#"); end >= 0 { - authority = authority[:end] - } - at := strings.LastIndex(authority, "@") - if at < 0 { - return "" - } - _, password, ok := strings.Cut(authority[:at], ":") - if !ok { - return "" - } - return password -} diff --git a/cmd/bd/telemetry_scrub_test.go b/cmd/bd/telemetry_scrub_test.go index dfce9f013..740746cb0 100644 --- a/cmd/bd/telemetry_scrub_test.go +++ b/cmd/bd/telemetry_scrub_test.go @@ -7,10 +7,12 @@ import ( "github.com/spf13/cobra" ) -// TestScrubArgsForTelemetry locks the invariant that a positional connection -// string cannot leak an embedded password into the bd.args telemetry span. This -// defense is backend-neutral and remains useful even when bd does not implement -// the connection string's database backend. +// TestScrubArgsForTelemetry locks the invariant that no password embedded in a +// --pg-url/--mysql-url value survives into the bd.args telemetry span, across every +// DSN password form pgx/go-sql-driver accept and both the --flag=value and +// --flag value spellings. The pre-fix scrubber handled only URL userinfo, so +// query-param and libpq keyword/value passwords (the confirmed red-team vectors) +// leaked verbatim. func TestScrubArgsForTelemetry(t *testing.T) { const secret = "s3cr3t-pw" cases := []struct { @@ -19,45 +21,35 @@ func TestScrubArgsForTelemetry(t *testing.T) { keep []string // non-secret structure that must survive redaction }{ { - name: "url userinfo", - argv: []string{"command", "postgres://bts:" + secret + "@127.0.0.1:5432/db"}, - keep: []string{"postgres://bts:", "127.0.0.1:5432/db"}, + name: "pg url userinfo =form", + argv: []string{"init", "--backend=postgres", "--pg-url=postgres://bts:" + secret + "@127.0.0.1:5432/db"}, + keep: []string{"--pg-url=postgres://bts:", "127.0.0.1:5432/db", "--backend=postgres"}, }, { - name: "url query param", - argv: []string{"command", "postgres://bts@127.0.0.1:5432/db?password=" + secret}, + name: "pg url query param space form", + argv: []string{"init", "--pg-url", "postgres://bts@127.0.0.1:5432/db?password=" + secret}, keep: []string{"127.0.0.1:5432/db", "password="}, }, { - name: "url sslpassword", - argv: []string{"command", "postgres://bts@h:5432/db?sslpassword=" + secret + "&sslmode=require"}, + name: "pg url sslpassword =form", + argv: []string{"init", "--pg-url=postgres://bts@h:5432/db?sslpassword=" + secret + "&sslmode=require"}, keep: []string{"sslmode=require"}, }, { - name: "keyword value connection string", - argv: []string{"command", "host=127.0.0.1 user=bts password=" + secret + " dbname=db"}, + name: "pg libpq keyword/value =form single token", + argv: []string{"init", "--pg-url=host=127.0.0.1 user=bts password=" + secret + " dbname=db"}, keep: []string{"host=127.0.0.1", "user=bts", "dbname=db"}, }, { - name: "keyword value connection string with whitespace around equals", - argv: []string{"command", "host = 127.0.0.1 password=" + secret}, - keep: []string{"host = 127.0.0.1", "password="}, - }, - { - name: "password-only keyword connection string", - argv: []string{"command", "password=" + secret}, - keep: []string{"password="}, - }, - { - name: "service keyword connection string", - argv: []string{"command", "service=production password=" + secret}, - keep: []string{"service=production", "password="}, - }, - { - name: "scheme-less userinfo", - argv: []string{"command", "bts:" + secret + "@tcp(127.0.0.1:3306)/db"}, + name: "mysql userinfo space form", + argv: []string{"init", "--mysql-url", "bts:" + secret + "@tcp(127.0.0.1:3306)/db"}, keep: []string{"tcp(127.0.0.1:3306)/db", "bts:"}, }, + { + name: "mysql userinfo =form", + argv: []string{"init", "--mysql-url=bts:" + secret + "@tcp(127.0.0.1:3306)/db"}, + keep: []string{"--mysql-url=bts:", "tcp(127.0.0.1:3306)/db"}, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -86,24 +78,11 @@ func TestScrubArgsForTelemetryLeavesOrdinaryArgs(t *testing.T) { if out := scrubArgsForTelemetry(argv, nil); out != "create --title document the password= knob" { t.Fatalf("over-redacted ordinary args: got %q", out) } - argv = []string{"create", "--title", "transport=rail password=foo"} - if out := scrubArgsForTelemetry(argv, nil); out != "create --title transport=rail password=foo" { - t.Fatalf("over-redacted non-DSN key/value text: got %q", out) - } argv = []string{"weird", "postgres://u:leak@h:5432/db"} if out := scrubArgsForTelemetry(argv, nil); strings.Contains(out, "leak") { t.Fatalf("userinfo password not scrubbed as defense in depth: %q", out) } - - for _, arg := range []string{ - "postgres://u@h:5432/db?password=leak", - "host=h user=u password=leak dbname=db", - } { - if out := scrubArgsForTelemetry([]string{"weird", arg}, nil); strings.Contains(out, "leak") { - t.Fatalf("positional DSN password not scrubbed as defense in depth: %q", out) - } - } } // TestSecretFlagTokensResolvesByCommand proves -p is treated as secret only on the @@ -211,14 +190,6 @@ func TestScrubArgsForTelemetryRedactsShorthandCluster(t *testing.T) { name: "boolean shorthand -v then -p cluster", argv: []string{"federation", "add-peer", "partner", "h:3306/db", "-vp" + secret}, }, - { - name: "boolean shorthand -q then separate -p value", - argv: []string{"federation", "add-peer", "partner", "h:3306/db", "-qp", secret}, - }, - { - name: "boolean shorthand -v then separate -p value", - argv: []string{"federation", "add-peer", "partner", "h:3306/db", "-vp", secret}, - }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -249,194 +220,3 @@ func TestScrubArgsForTelemetryKeepsOverloadedShortFlag(t *testing.T) { t.Fatalf("over-redacted non-secret -p=prefix: got %q", out) } } - -// TestTelemetryRedactionURLInFlagEqualsValue covers URL values embedded in a -// --flag=value argv element. The flag prefix must not prevent URL query parsing. -func TestTelemetryRedactionURLInFlagEqualsValue(t *testing.T) { - const secret = "query-secret" - arg := "--remote=postgres://bts@db.example/beads?password=" + secret - out := scrubArgsForTelemetry([]string{"init", arg}, nil) - if strings.Contains(out, secret) { - t.Fatalf("PASSWORD LEAK: URL query password in --flag=value survived telemetry scrub: %q", out) - } -} - -// TestTelemetryRedactionMalformedURLQuery fails closed when a URL-shaped DSN has -// an invalid percent escape. net/url rejects the whole URL in that case, but the -// query password must still never reach the bd.args telemetry span. -func TestTelemetryRedactionMalformedURLQuery(t *testing.T) { - const secret = "query-secret" - arg := "postgres://bts@db.example/beads%zz?password=" + secret + "&sslmode=require" - out := scrubArgsForTelemetry([]string{"command", arg}, nil) - if strings.Contains(out, secret) { - t.Fatalf("PASSWORD LEAK: malformed URL query password survived telemetry scrub: %q", out) - } - if !strings.Contains(out, "password=xxxxx") || !strings.Contains(out, "sslmode=require") { - t.Fatalf("malformed URL query structure was not preserved after redaction: %q", out) - } -} - -// TestScrubArgsForTelemetryAdversarialDSNVectors ports the adversarial regression -// vectors from the deleted internal/storage/pgdialect redaction tests -// (redact_error_test.go, redact_matrix_test.go). Each vector was a real leak or a -// scrubber-bypass near miss in an earlier redactor, so they are pinned here against -// the argv scrubber β€” the telemetry sink is backend-neutral and keeps receiving -// these DSN shapes even though the backends that motivated them are gone. -// -// Historical failure modes locked in, per case: -// - percent-encoded userinfo/query passwords must vanish in BOTH the raw -// (SUPER%2ASECRET) and decoded (SUPER*SECRET) forms; -// - percent-encoded query KEYS (pass%77ord, sslpass%77ord, %70assword) decode to -// password keys and bypassed key matching done on the raw key only; -// - password=SUPER\ SECRET: pgx accepts the backslash-escaped space as part of a -// single value, but a \S+-shaped value regex stopped at the space and shipped -// with the tail " SECRET" left in cleartext; -// - a single-quoted libpq password containing spaces is one value, not several; -// - pgx treats \v as inter-token whitespace, but Go's regexp \s class does not -// include \v, so a (^|\s)-anchored password regex never matched after \v and the -// password leaked entirely; -// - when one collected secret is a prefix of another (userinfo "foo", query -// "fooACTUAL"), replacing the short secret first turned "fooACTUAL" into -// "xxxxxACTUAL" and the tail survived; replacement must be length-descending. -func TestScrubArgsForTelemetryAdversarialDSNVectors(t *testing.T) { - const secret = "SUPERSECRET" - cases := []struct { - name string - dsn string - leaks []string // every cleartext form that must NOT survive redaction - keep []string // non-secret structure that must survive redaction - }{ - { - name: "url userinfo percent-encoded password", - dsn: "postgres://u:SUPER%2ASECRET@h:5432/db", - leaks: []string{"SUPER%2ASECRET", "SUPER*SECRET"}, - keep: []string{"postgres://u:", "h:5432/db"}, - }, - { - name: "url userinfo percent-encoded special chars", - dsn: "postgres://bts:p%40ss%3Aw%2Frd-" + secret + "@127.0.0.1:5432/db", - leaks: []string{"p%40ss%3Aw%2Frd-" + secret, "p@ss:w/rd-" + secret}, - keep: []string{"postgres://bts:", "127.0.0.1:5432/db"}, - }, - { - name: "url query param percent-encoded value", - dsn: "postgres://u@h:5432/db?password=SUPER%2ASECRET", - leaks: []string{"SUPER%2ASECRET", "SUPER*SECRET"}, - keep: []string{"password=", "h:5432/db"}, - }, - { - name: "url query param percent-encoded key (pass%77ord)", - dsn: "postgres://u@h:5432/db?pass%77ord=" + secret, - leaks: []string{secret}, - keep: []string{"pass%77ord=", "h:5432/db"}, - }, - { - name: "url query sslpassword percent-encoded key (sslpass%77ord)", - dsn: "postgres://u@h:5432/db?sslpass%77ord=" + secret + "&sslmode=require", - leaks: []string{secret}, - keep: []string{"sslpass%77ord=", "sslmode=require"}, - }, - { - name: "url query param fully percent-encoded key (%70assword)", - dsn: "postgres://u@h:5432/db?%70assword=" + secret, - leaks: []string{secret}, - keep: []string{"%70assword=", "h:5432/db"}, - }, - { - name: "url query password with sslmode preserved", - dsn: "postgres://bts@127.0.0.1:5432/db?password=" + secret + "&sslmode=disable", - leaks: []string{secret}, - keep: []string{"sslmode=disable", "127.0.0.1:5432/db"}, - }, - { - name: "url userinfo password with sslmode and application_name preserved", - dsn: "postgres://bts:" + secret + "@127.0.0.1:5432/db?sslmode=require&application_name=beads", - leaks: []string{secret}, - keep: []string{"sslmode=require", "application_name=beads", "127.0.0.1:5432/db"}, - }, - { - name: "url userinfo password but no user", - dsn: "postgres://:" + secret + "@h:5432/db", - leaks: []string{secret}, - keep: []string{"h:5432/db"}, - }, - { - name: "keyword value password with backslash-escaped whitespace", - dsn: `host=h user=u password=SUPER\ SECRET dbname=db`, - // "SECRET" pins the historical shipped bug: the tail after the escaped - // space must not survive in cleartext. - leaks: []string{"SECRET", `SUPER\`}, - keep: []string{"host=h", "user=u", "dbname=db"}, - }, - { - name: "keyword value single-quoted password containing spaces", - dsn: "host=h user=u password='se cret " + secret + "' dbname=db", - leaks: []string{"se cret", secret}, - keep: []string{"host=h", "user=u", "dbname=db"}, - }, - { - name: "keyword value single-quoted password", - dsn: "host=h user=u password='" + secret + "' dbname=db", - leaks: []string{secret}, - keep: []string{"host=h", "user=u", "dbname=db"}, - }, - { - name: "keyword value vertical-tab separator", - dsn: "host=h user=u\vpassword=" + secret + " dbname=db", - leaks: []string{secret}, - keep: []string{"host=h", "dbname=db"}, - }, - { - name: "keyword value sslpassword with sslmode preserved", - dsn: "host=h user=u sslpassword=" + secret + " sslmode=require dbname=db", - leaks: []string{secret}, - keep: []string{"host=h", "sslmode=require", "dbname=db"}, - }, - { - name: "keyword value password but no user", - dsn: "host=h port=5432 password=" + secret + " dbname=db", - leaks: []string{secret}, - keep: []string{"host=h", "port=5432", "dbname=db"}, - }, - { - name: "overlapping secrets where one is a prefix of another", - dsn: "postgres://u:foo@h:5432/db?password=fooACTUAL", - leaks: []string{"ACTUAL", ":foo@"}, - keep: []string{"postgres://u:", "h:5432/db"}, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - out := scrubArgsForTelemetry([]string{"command", tc.dsn}, nil) - for _, leak := range tc.leaks { - if strings.Contains(out, leak) { - t.Errorf("PASSWORD LEAK: %q survived telemetry scrub of %q: %q", leak, tc.dsn, out) - } - } - if !strings.Contains(out, "xxxxx") { - t.Errorf("expected redaction marker xxxxx in %q", out) - } - for _, k := range tc.keep { - if !strings.Contains(out, k) { - t.Errorf("expected %q to survive redaction in %q", k, out) - } - } - }) - } -} - -// TestScrubArgsForTelemetryPasswordlessDSNUnchanged ports the deleted matrix -// round-trip invariant: a DSN carrying no password must pass through the scrubber -// byte-for-byte unchanged, in both URL and libpq keyword/value form. Over-redaction -// here would destroy the non-secret operational signal the span exists to carry. -func TestScrubArgsForTelemetryPasswordlessDSNUnchanged(t *testing.T) { - for _, dsn := range []string{ - "postgres://bts@127.0.0.1:5432/db?application_name=beads&sslmode=require", - "host=127.0.0.1 port=5432 user=bts dbname=db sslmode=require", - } { - want := "command " + dsn - if out := scrubArgsForTelemetry([]string{"command", dsn}, nil); out != want { - t.Errorf("passwordless DSN altered by scrubber:\n in = %q\n out = %q", want, out) - } - } -} diff --git a/cmd/bd/version_tracking.go b/cmd/bd/version_tracking.go index 1ef580632..acba00814 100644 --- a/cmd/bd/version_tracking.go +++ b/cmd/bd/version_tracking.go @@ -163,10 +163,6 @@ func autoMigrateOnVersionBump(beadsDir string) { if cfg == nil { cfg = configfile.DefaultConfig() } - if cfg.GetBackend() != configfile.BackendDolt { - debug.Logf("auto-migrate: skipping Dolt migration for backend %q", cfg.GetBackend()) - return - } if cfg.IsDoltProxiedServerMode() { debug.Logf("auto-migrate: skipping embedded migration, proxied-server handled after UOW provider init") diff --git a/cmd/bd/version_tracking_test.go b/cmd/bd/version_tracking_test.go index fa46d271b..01678adef 100644 --- a/cmd/bd/version_tracking_test.go +++ b/cmd/bd/version_tracking_test.go @@ -434,5 +434,5 @@ func TestAutoMigrateOnVersionBump_NoDatabase(t *testing.T) { // TestAutoMigrateOnVersionBump_RefusesDowngrade, and TestAutoMigrateOnVersionBump_TracksMaxVersion // were removed because they depended on the SQLite storage backend (sqlite.New) for round-trip // persistence testing through autoMigrateOnVersionBump -> dolt.NewFromConfig. -// Automatic schema migration is exercised here through the Dolt backend. +// Since SQLite support has been removed, auto-migration is only exercised with Dolt backends. // Dolt-based migration testing is covered by TestInitDoltMetadata in init_test.go. diff --git a/default.nix b/default.nix index aef41c212..bb503ed7c 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,7 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-L6pzbuY6h0FXYBTk5rvIUTSI2tsYthoNb1N7h3hWvZs="; + vendorHash = "sha256-Ij4xnWQLre/uE48TGMJRSGrunLqfXQ8MVBA+NCKlhLE="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index 2c2e52ca8..9cb0177ef 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -3543,7 +3543,7 @@ Initialize bd in the current directory by creating a .beads/ directory and its storage (a Dolt database by default). Optionally specify a custom issue prefix. Dolt is the default backend and the only one with version control (history, -branching, sync). Select the embedded SQLite alternative with --backend=sqlite; +branching, sync). Select an alternative with --backend=<postgres|mysql|sqlite>; see docs/architecture/storage-backends.md for the trade-offs and setup. Use --database to specify an existing server database name, overriding the @@ -3584,7 +3584,7 @@ bd init [flags] --agents-file string Custom filename for agent instructions (default: AGENTS.md) --agents-profile string AGENTS.md profile: 'minimal' (default, pointer to bd prime) or 'full' (complete command reference) --agents-template string Path to custom AGENTS.md template (overrides embedded default) - --backend string Storage backend: dolt (default) or sqlite. See docs/architecture/storage-backends.md. + --backend string Storage backend: dolt (default), postgres, mysql, or sqlite. See docs/architecture/storage-backends.md. --contributor Run OSS contributor setup wizard --database string Use existing server database name (overrides prefix-based naming) --debug Run the managed Dolt sql-server with --loglevel=debug and CPU profiling (--prof cpu). Persisted to config.yaml as dolt.debug. No effect on externally-managed servers. @@ -3594,7 +3594,11 @@ bd init [flags] --force Deprecated alias for --reinit-local. Bypasses only the LOCAL data-safety guard; does NOT authorize remote divergence (see 'bd help init-safety'). --from-jsonl Import issues from configured import.path; refuses remote history unless --discard-remote authorizes replacement --init-if-missing If the workspace is already initialized, skip init and exit 0 instead of failing (idempotent init for scaffolds) + --mysql-database string MySQL database for this workspace (with --backend=mysql; MySQL's isolation unit) + --mysql-url string MySQL server DSN (with --backend=mysql), e.g. user:pass@tcp(host:3306)/ . A password may be included for init but is never persisted; set BEADS_MYSQL_PASSWORD for later commands. Falls back to BEADS_MYSQL_URL. --non-interactive Skip all interactive prompts (auto-detected in CI or non-TTY environments) + --pg-schema string Postgres schema for this workspace's tables (with --backend=postgres; provides search_path isolation) + --pg-url string Postgres connection URL (with --backend=postgres). A password may be included for init but is never persisted; set BEADS_PG_PASSWORD for later commands. Falls back to BEADS_POSTGRES_URL. -p, --prefix string Issue prefix (default: current directory name) --proxied-server [EXPERIMENTAL] Use a per-workspace proxied dolt sql-server (proxy + child dolt) rooted at .beads/dolt --proxied-server-config-path string [EXPERIMENTAL] Absolute path to an existing dolt sql-server YAML config (proxied-server mode only). When set, bd uses this file instead of auto-generating one. Relative paths are rejected. @@ -4032,13 +4036,6 @@ This command checks: - .beads/.gitignore up to date - Metadata.json version tracking (LastBdVersion field) -Storage Availability: - Full diagnostics, --perf, --deep, --server, --migration, and - --check=validate currently require Dolt server mode. Embedded Dolt and - SQLite support --check=artifacts, --check=conventions, and - --check=pollution. --check-health has a limited hook-health fallback. - Unsupported combinations return a notice without changing storage. - Performance Mode (--perf): Run performance diagnostics on your database: - Times key operations (bd ready, bd list, bd show, etc.) @@ -4078,12 +4075,18 @@ Server Mode (--server): - Schema compatible: Can query beads tables? - Connection pool: Pool health metrics -Legacy Dolt Migration Validation Mode (--migration): - Retained for older SQLite-to-Dolt migration workflows and available only in - Dolt server mode. It does not inspect or migrate a current SQLite workspace, - and it is not the migration path for removed PostgreSQL or MySQL backends. - Use the storage-backends migration guide for current backend moves. Combine - with --json for machine-parseable diagnostic output. +Migration Validation Mode (--migration): + Run Dolt migration validation checks with machine-parseable output. + Use --migration=pre before migration to verify readiness: + - JSONL file exists and is valid (parseable, no corruption) + - All JSONL issues are present in SQLite (or explains discrepancies) + - No blocking issues prevent migration + Use --migration=post after migration to verify completion: + - Dolt database exists and is healthy + - All issues from JSONL are present in Dolt + - No data was lost during migration + - Dolt database has no locks or uncommitted changes + Combine with --json for machine-parseable output for automation. Agent Mode (--agent): Output diagnostics designed for AI agent consumption. Instead of terse @@ -4130,9 +4133,9 @@ Examples: bd doctor --check=validate --fix # Auto-fix data-integrity issues bd doctor --deep # Full graph integrity validation bd doctor --server # Dolt server mode health checks - bd doctor --migration=pre # Legacy Dolt-server migration diagnostic - bd doctor --migration=post # Legacy Dolt-server completion diagnostic - bd doctor --migration=pre --json # Machine-parseable legacy diagnostic + bd doctor --migration=pre # Validate readiness for Dolt migration + bd doctor --migration=post # Validate Dolt migration completed + bd doctor --migration=pre --json # Machine-parseable migration validation ``` bd doctor [path] [flags] @@ -4150,7 +4153,7 @@ bd doctor [path] [flags] --fix Automatically fix issues where possible --fix-child-parent Remove childβ†’parent dependencies (opt-in) -i, --interactive Confirm each fix individually - --migration string Run legacy Dolt-server migration diagnostics: 'pre' or 'post' + --migration string Run Dolt migration validation: 'pre' (before migration) or 'post' (after migration) --orchestrator Running in orchestrator multi-workspace mode (routes.jsonl is expected, higher duplicate tolerance) --orchestrator-duplicates-threshold int Duplicate tolerance threshold for orchestrator mode (wisps are ephemeral) (default 1000) -o, --output string Export diagnostics to JSON file diff --git a/docs/architecture/dolt.md b/docs/architecture/dolt.md index fb662a4e4..9bb7fde98 100644 --- a/docs/architecture/dolt.md +++ b/docs/architecture/dolt.md @@ -3,10 +3,7 @@ title: Dolt Backend for Beads description: "How beads uses Dolt for versioned issue storage: embedded vs server mode, remotes, sync, and backups" --- -Dolt is Beads' default storage backend. It provides a version-controlled SQL -database with cell-level merge, native branching, and two deployment modes. -For the supported server-free SQLite alternative, see -[Storage Backends](/architecture/storage-backends). +Beads uses Dolt as its storage backend. Dolt provides a version-controlled SQL database with cell-level merge, native branching, and two deployment modes. ## Why Dolt? @@ -46,11 +43,9 @@ gt dolt start # Start the Dolt server bd init --server # Initialize with server mode ``` -### Migrate from Legacy SQLite Installations +### Migrate from SQLite (Legacy) -Current releases support SQLite through the storage interface. This section is -only for upgrades from older, pre-Dolt SQLite schemas that are not compatible -with the current SQLite implementation. +If upgrading from an older version that used SQLite: > **Note:** The `bd migrate --to-dolt` command was removed in v0.58.0. > For pre-0.50 installations with JSONL data, use the migration script: @@ -447,7 +442,6 @@ dolt: | Variable | Purpose | |----------|---------| | `BEADS_DOLT_PASSWORD` | Server mode password (highest priority) | -| `BEADS_DOLT_CREDENTIAL_COMMAND` | Gateway identity command (token becomes the connection username) | | `BEADS_CREDENTIALS_FILE` | Path to credentials file (overrides default location) | | `BEADS_DOLT_SERVER_MODE` | Enable server mode (set to "1") | | `BEADS_DOLT_SERVER_HOST` | Server host (default: 127.0.0.1) | @@ -459,31 +453,6 @@ dolt: | `DOLT_REMOTE_PASSWORD` | Clone/push/pull auth password | | `BD_DOLT_AUTO_COMMIT` | Override auto-commit setting | -### Gateway Identity Credentials - -Authenticating gateway deployments can set -`BEADS_DOLT_CREDENTIAL_COMMAND` to a command that mints a short-lived identity -token. Beads runs the command, reads a bare token or supported JSON token -envelope from stdout, and presents that token as the MySQL connection username. -The gateway verifies the identity and routes the connection to the appropriate -Dolt database. - -This is an identity credential, not a database password: - -- A server user already preset by the opener takes precedence. Otherwise the - command runs before the static `BEADS_DOLT_SERVER_USER` or metadata user. -- A configured command that fails aborts the connection. Beads does not fall - back to the static `root` user. -- The command is read from the environment only and is never persisted in - workspace metadata. -- Gateway mode assumes an externally managed server, so Beads does not - auto-start a local `dolt sql-server`. -- Tokens containing `:`, `@`, or `/` are rejected because the MySQL DSN grammar - cannot represent them safely in the username field. - -For ordinary Dolt server deployments, use `BEADS_DOLT_PASSWORD` or the -credentials file below instead. - ### Credentials File For multi-server setups, you can store passwords in an INI-style credentials file diff --git a/docs/architecture/index.md b/docs/architecture/index.md index d20a865e2..d8c347b0f 100644 --- a/docs/architecture/index.md +++ b/docs/architecture/index.md @@ -1,21 +1,13 @@ --- title: Architecture Overview -description: "How Beads stores and queries issue data, with Dolt-backed history and sync" +description: "How Beads stores, queries, and syncs issue data with Dolt" --- -This document explains the default Dolt architecture: storage layout, data -model, and sync paths. Beads also supports SQLite behind the same storage -interface; see [Storage Backends](/architecture/storage-backends) for the -supported paths and their trade-offs. For the concept model β€” beads, -dependencies, ready work, molecules β€” see -[How Beads Works](/core-concepts/index). +This document explains how Beads' architecture works with Dolt as its storage backend: the storage layout, the data model, and the sync paths. For the concept model β€” beads, dependencies, ready work, molecules β€” see [How Beads Works](/core-concepts/index). ## Architecture -Beads uses **Dolt** by default. Dolt is a version-controlled SQL database that -provides git-like semantics (branch, merge, diff, push, pull) natively at the -database level. SQLite is the supported server-free alternative when those -version-control capabilities are not needed. +Beads uses **Dolt** as its sole storage backend -- a version-controlled SQL database that provides git-like semantics (branch, merge, diff, push, pull) natively at the database level. By default, Dolt runs in **embedded mode** (in-process, no separate server). For multi-writer setups (multiple agents, orchestrator), switch to **server mode** which connects to a @@ -278,7 +270,7 @@ For these use cases, consider GitHub Issues, Linear, or Jira. - [How Beads Works](/core-concepts/index) β€” The concept model: beads, dependencies, ready work, molecules - [Sync Concepts](/core-concepts/sync-concepts) β€” Cross-machine sync, wire format, and anti-patterns - [Dolt Backend](/architecture/dolt) β€” Embedded vs server mode in depth, shared server, migration -- [Storage Backends](/architecture/storage-backends) β€” Embedded Dolt, Dolt server, and SQLite +- [Storage Backends](/architecture/storage-backends) β€” Postgres, MySQL, and SQLite alternatives - [Recovery Runbooks](/recovery/index) β€” Step-by-step procedures for common issues - [CLI Reference](/cli-reference/index) β€” Complete command documentation - [Getting Started](/index) β€” Installation and first steps diff --git a/docs/architecture/storage-backends.md b/docs/architecture/storage-backends.md index f07179131..ce7a6102c 100644 --- a/docs/architecture/storage-backends.md +++ b/docs/architecture/storage-backends.md @@ -1,118 +1,101 @@ --- title: Storage Backends -description: Choosing between embedded Dolt, Dolt server, and SQLite, and understanding which version-control features each path provides +description: Running bd on Dolt, Postgres, MySQL, or SQLite, and what version-control features each backend keeps --- -Beads keeps issue storage behind one interface and supports three deployment -paths: +Beads uses Dolt as its default storage backend, and Dolt remains the reference implementation. You can also point `bd` at Postgres, MySQL, or SQLite β€” every backend implements the full issue-tracking core, and the only thing you give up off Dolt is version control. -- **Embedded Dolt (default)** β€” version-controlled storage in the `bd` process -- **Dolt server** β€” the same version-controlled storage through a - `dolt sql-server`, for concurrent writers -- **SQLite** β€” a small, server-free database file for local and resource-light - workspaces - -The issue-tracking semantics are written once in the shared core. Dolt is the -reference implementation and adds history, branching, remotes, and sync. -SQLite runs the same issue core through the SQL adapter and returns a clear, -typed error for operations that require Dolt's commit graph. +One seam, one shared core: the issue-tracking semantics are written once (in `issueops`) and shared by *every* backend β€” Dolt runs them natively, the SQL family runs them through one thin per-engine dialect. Each backend is a store adapter over that shared core. Version control lives only in the Dolt store; on the SQL backends it returns one clean, typed error. ```mermaid flowchart TD - CLI["bd CLI"] --> SEAM["Storage interface
one seam for every implementation"] + CLI["bd CLI"] --> SEAM["Storage interface
one seam β€” every backend satisfies all of it"] - SEAM --> CORE["Issue core
CRUD Β· search Β· deps Β· ready/claim Β· labels Β· comments"] - SEAM --> CAPS["Dolt capabilities
history Β· branch Β· remotes Β· sync"] + SEAM --> CORE["CORE β€” issueops
CRUD Β· search Β· deps Β· ready/claim Β· ordering Β· id-mint
written once, shared by EVERY backend"] + SEAM --> CAPS["CAPABILITIES β€” version control
history Β· branch Β· remotes Β· sync"] - CORE --> DS["Dolt store"] - CORE --> KIT["sqlkit.Store"] --> SQLITE[("SQLite file")] + CORE --> DS["Dolt store
runs the core natively"] + CORE --> KIT["sqlkit.Store
SQL-family adapter Β· Dialect / Readiness / Claim"] - DS --> EMBED[("Embedded Dolt")] - DS --> SERVER[("Dolt SQL server")] - CAPS --> DS - CAPS -->|"typed unsupported error"| SQLITE + DS --> DOLT[("Dolt β€” embedded / server
the conformance ORACLE")] + KIT --> PGD["pgdialect"] --> PG[("Postgres")] + KIT --> MYD["mysqldialect"] --> MY[("MySQL")] + KIT --> LTD["sqlitedialect"] --> LT[("SQLite")] - DS -. "conformance reference" .-> KIT + CAPS -->|"implemented in the Dolt store"| DS + CAPS -->|"generated typed-unsupported shell"| SHELL["Postgres / MySQL / SQLite
operation X not supported"] + + DOLT -. "conformance-tested: SQL family matches Dolt byte-for-byte" .-> KIT ``` -```text +``` bd CLI | Storage interface | - +------------------+------------------+ - | | - Issue core Dolt capabilities - CRUD, search, deps, claims, history, branch, remotes, - labels, comments, ordering sync - | | - +------+-------+ | - | | | - Dolt store sqlkit.Store <--- conformance ----+ - | | - +---+---+ SQLite file - | | (typed unsupported errors for -embedded server Dolt-only capabilities) - Dolt Dolt + +-------------------------+--------------------------+ + | | + CORE (every backend) CAPABILITIES (Dolt-only) + issueops -- CRUD, search, deps, version control: + ready/claim, ordering, id-mint, history, branch, remotes, sync + cycle checks: written ONCE | + | +-> Dolt store: implemented + +--------------------+ +-> Postgres/MySQL/SQLite: + | | generated typed-"unsupported" + Dolt store sqlkit.Store (operation "X" not supported) + runs the core SQL-family adapter + natively (Dialect/Readiness/Claim) + | | + [Dolt engine] +------+------+---------+ + embedded/server pgdialect mysqldialect sqlitedialect + = conformance | | | + ORACLE Postgres MySQL SQLite + + issueops is shared by EVERY backend (Dolt included); each backend is a store + adapter over it. Dolt runs the core's SQL natively and adds version control; + the SQL family runs it through one thin dialect per engine, and is + conformance-tested to match the Dolt oracle byte-for-byte. ``` -## Supported Scope +## Why Choose a Backend? -The recently added direct PostgreSQL and MySQL adapters have been rolled back. -Supporting additional general-purpose server databases adds dialect, -credential, schema-lifecycle, migration, CI, and operational complexity. Our -goal is to keep Beads as simple as possible and consume as few resources as -possible, so the supported implementations stay focused on Dolt and SQLite. +- **Dolt stays the default** β€” `bd init` with no flags behaves exactly as before, and Dolt is the only backend with history, branching, and sync +- **Postgres / MySQL** β€” put your issues in the database your team already runs, backs up, and monitors; many writers, one server +- **SQLite** β€” a single file inside `.beads/`, zero servers, zero credentials, pure-Go driver +- **Same `bd` everywhere** β€” create, list, ready, deps, claims, labels, comments, search all behave identically on every backend; conformance tests enforce it against the Dolt reference +- **Clean degradation** β€” Dolt-only commands don't half-work on SQL backends; they return one clear, typed error -This does **not** remove Dolt server mode. A `dolt sql-server` speaks the MySQL -wire protocol, and Beads continues to support that mode for concurrent writers. -A MySQL-protocol reference in the Dolt documentation therefore describes a -Dolt connection, not a generic MySQL storage backend. +## Pick Your Backend -## Choose a Storage Path - -| Storage path | Best for | History and sync | Concurrent writers | Server or credentials | +| Backend | Best for | History | Server needed | Credentials | |---|---|---|---|---| -| Embedded Dolt (default) | Most projects; offline-first work with versioned issue data | Yes | One process at a time | None | -| Dolt server | Teams or orchestrators with multiple writers | Yes | Yes | `dolt sql-server`; credentials when configured | -| SQLite | Local work, CI sandboxes, air-gapped or minimal-footprint environments | No | Writes are serialized | None | +| `dolt` (default) | Solo or team work where you want issue history, branching, `bd dolt push`/`pull` | Yes | No (embedded) / optional | Only in server mode | +| `sqlite` | Solo work, throwaway workspaces, CI sandboxes, air-gapped machines | No | No | None | +| `postgres` | Teams with an existing Postgres; many workspaces share one database via per-workspace schemas | No | Yes | Ladder (below) | +| `mysql` | Teams with an existing MySQL; each workspace gets its own database | No | Yes | Ladder (below) | Rules of thumb: -- Use **embedded Dolt** unless you have a reason to choose differently. -- Use **Dolt server** when multiple processes need to write concurrently. -- Use **SQLite** when the smallest server-free footprint matters more than - history, branching, federation, or cross-machine sync. +- If you want `bd history`, `bd dolt push`, time-travel, or federation β€” use **Dolt**. It is the only backend that tracks history. +- If you want the smallest possible footprint and don't need history β€” use **SQLite**. +- If your organization already operates Postgres or MySQL and you want issues living next to everything else your DBAs manage β€” use **Postgres** or **MySQL**. Multi-writer concurrency comes from the database server itself. +- If in doubt, use the default. `bd init` without `--backend` is Dolt, unchanged. ## Getting Started -The backend is selected at `bd init` time and recorded in -`.beads/metadata.json`. Later commands reopen that implementation -automatically. +The backend is chosen once, at `bd init` time, and recorded in `.beads/metadata.json`. Every later `bd` command reads it from there β€” no per-command flags. -### Embedded Dolt (default) +### Dolt (default) ```bash bd init --prefix myproj ``` -Dolt runs in-process and stores data under `.beads/embeddeddolt/`. No separate -server or credentials are needed. - -### Dolt server - -```bash -bd init --server --prefix myproj -``` - -Server mode connects to a `dolt sql-server`. Managed repo-local mode stores -data under `.beads/dolt/`; shared, external, and gateway servers manage their -own data locations. See [Dolt Backend for Beads](/architecture/dolt) for -managed, shared, external, socket, TLS, and credential configuration. +Nothing changes. See [Dolt Backend for Beads](/architecture/dolt) for embedded vs. server mode. ### SQLite @@ -123,109 +106,174 @@ bd init --backend=sqlite --prefix myproj bd init --backend=sqlite --sqlite-path=issues.db --prefix myproj ``` -SQLite needs no server and no credentials. Foreign keys and transaction -locking are configured automatically. +That's it β€” no server, no credentials. Foreign keys and immediate-transaction locking are configured automatically. -## Credentials +### Postgres -Embedded Dolt and SQLite need no database credentials. Dolt server mode can -use `BEADS_DOLT_PASSWORD` or the per-host credentials file documented in the -[Dolt backend guide](/architecture/dolt#credentials-file). +Each Beads workspace lives in its own Postgres **schema**, so many workspaces can share one database. -Authenticating gateway deployments can instead set -`BEADS_DOLT_CREDENTIAL_COMMAND`. The command produces a short-lived identity -token that Beads presents as the connection username to the gateway. This is a -Dolt server feature; see -[Gateway identity credentials](/architecture/dolt#gateway-identity-credentials) -for the fail-closed behavior and precedence rules. +```bash +bd init --backend=postgres \ + --pg-url='postgres://bd:onlyforinit@db.example.com:5432/beads?sslmode=require' \ + --pg-schema=myproj \ + --prefix myproj +``` -## Existing PostgreSQL or MySQL Workspaces +- `--pg-url` β€” a standard `postgres://` URL. If omitted, `bd init` falls back to the `BEADS_POSTGRES_URL` environment variable. +- `--pg-schema` β€” the schema this workspace owns; created and pinned via `search_path`. -Current Beads builds still recognize PostgreSQL and MySQL workspace markers so -they can stop safely with migration guidance. They do not connect to, open, or -modify the configured PostgreSQL or MySQL database. +> **Note:** `bd init` connects with the URL exactly as given, so embed the password +> in `--pg-url` for init. It is stripped before anything is written to disk β€” +> `.beads/metadata.json` stores only the password-free DSN β€” and every later +> command resolves the password through the credential ladder below. -To move an existing workspace: +```bash +# For all subsequent commands, supply the password via any ladder rung, e.g.: +export BEADS_PG_PASSWORD_COMMAND='vault kv get -field=password secret/beads/pg' -1. Leave the original server database and workspace metadata in place. Take a - server-native backup before making changes. -2. Before upgrading, or with a separate `bd` build that supports the configured - backend, create an issue export: +bd create "first issue" -p 1 +bd ready +``` - ```bash - bd export --all -o beads-export.jsonl - ``` +### MySQL -3. Install the current build and follow `bd help init-safety` before - reinitializing the workspace with embedded Dolt, Dolt server, or SQLite. -4. Import the issue export and verify the resulting workspace: +Each Beads workspace gets its own MySQL **database**. - ```bash - bd import beads-export.jsonl - ``` +```bash +bd init --backend=mysql \ + --mysql-url='bd:onlyforinit@tcp(db.example.com:3306)/' \ + --mysql-database=myproj_beads \ + --prefix myproj +``` -5. Keep the original server database until the imported workspace has been - checked and backed up using its new storage path. +- `--mysql-url` β€” go-sql-driver DSN grammar: `user:password@tcp(host:port)/` (note the trailing slash, no scheme). Falls back to `BEADS_MYSQL_URL` if omitted. +- `--mysql-database` β€” the database this workspace owns. -Do not edit `.beads/metadata.json` to point at another backend. JSONL is an -issue-level interchange format, not a complete database backup, and does not -replace the server-native backup from step 1. +Same rule as Postgres: embed the password in `--mysql-url` for init only; it is redacted before persisting, and later commands use the ladder: -## What SQLite Keeps and What It Gives Up +```bash +export BEADS_MYSQL_PASSWORD_COMMAND='vault kv get -field=password secret/beads/mysql' +# or a static password: +export BEADS_MYSQL_PASSWORD=... +``` -SQLite implements the complete issue-tracking core: +> **Note:** the MySQL DSN grammar cannot carry a password without a username. +> `bd` refuses such DSNs loudly rather than silently connecting passwordless. -- Issue CRUD, bulk creates, close, reopen, and delete -- Search, counts, filtering, and sorting -- Dependencies, cycle detection, and dependency trees -- Ready and blocked queries, claims, and leases -- Labels, comments, events, config, metadata, and statistics -- Transactions and streaming iterators +## Credentials & Override Layers -Operations built on Dolt's version-control capabilities remain Dolt-only: +SQLite needs no credentials. For Postgres and MySQL, the password is resolved fresh at open time by a **ladder** of sources, highest priority first. The first *configured* rung wins, and a configured rung that fails aborts the connection β€” it never silently falls through to a lower rung. -- **History and time travel** β€” `bd history`, as-of queries, and diffs -- **Branching and merging** β€” branch, checkout, merge, and conflict resolution -- **Remotes and sync** β€” `bd dolt push`, `bd dolt pull`, fetch, and sync +Precedence, per backend: + +1. **Password embedded in the DSN** β€” a password inside `BEADS_POSTGRES_URL` / `BEADS_MYSQL_URL` wins outright; no ladder runs. +2. **Credential command** β€” `BEADS_PG_PASSWORD_COMMAND` / `BEADS_MYSQL_PASSWORD_COMMAND` +3. **Static env var** β€” `BEADS_PG_PASSWORD` / `BEADS_MYSQL_PASSWORD` +4. **Credentials file** β€” an entry matching the DSN's `host:port` +5. **Driver-native fallback** β€” Postgres only: `PGPASSWORD`, `~/.pgpass`, `PGPASSFILE` still work when nothing above is configured. MySQL has no driver-native fallback; with nothing configured it attempts an empty password. + +> **Note the naming asymmetry:** the URL variables use full names +> (`BEADS_POSTGRES_URL`, `BEADS_MYSQL_URL`) while the password variables +> abbreviate Postgres (`BEADS_PG_PASSWORD`, `BEADS_PG_PASSWORD_COMMAND`). +> There is no `BEADS_PG_URL`. + +### The credential command (rotating secrets) + +This is the same idiom as kubectl's ExecCredential, the AWS CLI's `credential_process`, and git credential helpers: `bd` runs your command with `sh -c` (30-second timeout) and reads the secret from stdout. Output can be a bare token, a kubectl-style JSON envelope (`{"token": "...", "expirationTimestamp": "..."}`), or an OAuth-style one (`{"access_token": "...", "expires_in": 900}`). Results are cached in-process and refreshed shortly before expiry, so repeated opens don't re-spawn your helper. + +```bash +# Vault +export BEADS_PG_PASSWORD_COMMAND="vault kv get -field=password secret/beads/pg" + +# AWS RDS IAM auth (15-minute tokens; caching and expiry are handled for you) +export BEADS_PG_PASSWORD_COMMAND="aws rds generate-db-auth-token \ + --hostname db.example.com --port 5432 --username bd" +``` + +The command variables are read from the environment only β€” never from workspace metadata β€” because a persisted command would be arbitrary code executed on every `bd` invocation. + +### The credentials file (static, per-host) + +The lowest configured rung is an INI-style file keyed by `host:port`, at `~/.config/beads/credentials` (Linux/macOS) or `%APPDATA%\beads\credentials` (Windows); override the path with `BEADS_CREDENTIALS_FILE`. + +```ini +# ~/.config/beads/credentials (chmod 600) +[127.0.0.1:5432] +password=localDevPassword + +[db.example.com:3306] +password=teamServerPassword +``` + +If the file is readable by group or others, `bd` prints an ssh-style warning. The file rung is deliberately forgiving: a missing file or unmatched section just means "not configured" and the ladder continues β€” fail-closed behavior is reserved for rungs you explicitly configured, like a command. + +### Dolt server credentials + +The Dolt backend in server mode has its own rungs: `BEADS_DOLT_PASSWORD`, then the credentials file, with `BEADS_DOLT_SERVER_USER` overriding the username. There is also `BEADS_DOLT_CREDENTIAL_COMMAND`, an *identity* credential: the minted token is presented as the connection username to an authenticating gateway server, which verifies it and routes to the database. This is for gateway deployments only β€” the direct Postgres/MySQL backends reject `BEADS_PG_CREDENTIAL_COMMAND` / `BEADS_MYSQL_CREDENTIAL_COMMAND` with an error telling you to use the `_PASSWORD_COMMAND` form instead. + +### Secrets never reach disk + +`bd init` redacts the password from your URL before writing `.beads/metadata.json`, using the database driver's own parser, and then re-parses to verify nothing survived β€” refusing to init rather than persisting a secret it can't prove is gone. Check for yourself: + +```bash +cat .beads/metadata.json +# {"backend":"postgres","postgres_dsn":"postgres://bd@db.example.com:5432/beads?sslmode=require", ...} +``` + +## What You Keep and What You Give Up + +Every backend implements the complete issue-tracking core β€” the same shared implementation, not a re-derivation per backend: + +- Issue CRUD, bulk creates, close/reopen/delete +- Search, counts, filtering, sorting +- Dependencies, cycle detection, dependency trees +- Ready/blocked work queries, claims and leases +- Labels, comments, the full event/audit trail +- Config, metadata slots, statistics, transactions, streaming iterators + +What is Dolt-only β€” by design, since these *are* version control: + +- **History & time travel** β€” `bd history`, as-of queries, diffs +- **Branching & merging** β€” branch, checkout, merge, conflict resolution +- **Remotes & sync** β€” `bd dolt push` / `bd dolt pull`, fetch, sync - **Federation** β€” peer management - **Compaction** β€” snapshot-based memory compaction -Those operations fail explicitly on SQLite instead of approximating Dolt -behavior: +On Postgres, MySQL, and SQLite these operations don't limp along or approximate β€” each returns one clean, typed error: -```text +``` $ bd history myproj-9q9 Error: failed to get history: operation "History" not supported by the sqlite backend ``` -`bd init` also sets the expectation up front: +`bd init` sets the expectation up front: -```text +``` βœ“ bd initialized with the SQLite backend History: not tracked (SQLite backend has no version control; use Dolt for history) ``` -SQLite writes are durable when the command returns. The Dolt-only maintenance -tailβ€”history commits, native backups, and remote pushesβ€”is skipped. Use -`bd export` when you need a portable JSONL copy of issue data; JSONL remains an -interchange format rather than a full database backup. +A few practical consequences: -## How Conformance Is Enforced +- Writes on SQL backends are durable the moment the command returns β€” each write commits its own SQL transaction. (`bd`'s internal commit hooks become harmless no-ops, not errors.) +- The Dolt-only maintenance tail (auto-commit, auto-export, auto-backup, auto-push) is skipped automatically on non-Dolt backends. +- "Sync" on Postgres/MySQL is your database server: everyone connects to the same database, so there is nothing to push or pull. Back up issues with your normal database tooling, or `bd export` for JSONL interoperability. -The conformance harness holds SQLite to the embedded-Dolt reference: +## How It Stays Correct -- **Tier 1 (in-process)** runs the same issue-store behavior corpus against - embedded Dolt and SQLite, including the typed-unsupported contract for - Dolt-only operations. -- **Tier 2 (end-to-end)** builds a real `bd` binary, initializes isolated Dolt - and SQLite workspaces, and compares normalized CLI output. +The SQL backends are held to the embedded-Dolt reference by a conformance harness, run locally and in CI (with real Postgres and MySQL servers) on every pull request: -Run both tiers with: +- **Tier 1 (in-process):** the same behavior corpus the Dolt reference passes β€” CRUD, search, dependencies, ready/blocked, claims and leases, plus a ~94-case audit corpus of edge behaviors β€” runs against every backend. Two extra gates make capability gaps impossible to hide: a completeness test proves the set of unsupported operations exactly matches an audited allowlist, and a contract test calls every one of them to prove it returns the typed error rather than panicking or lying. +- **Tier 2 (end-to-end):** a real `bd` binary runs `bd init --backend=...` and a CLI scenario corpus in an isolated workspace per backend, and the normalized output must be byte-identical to the Dolt reference. Known-divergence entries are visible in the registry and can only shrink β€” currently there are none. ```bash ./scripts/conformance.sh +# Postgres/MySQL profiles self-skip unless you point them at a server: +BEADS_PG_TEST_URL="postgres://user:pass@127.0.0.1:5432/beads_test" \ +BEADS_MYSQL_TEST_URL="user:pass@tcp(127.0.0.1:3306)/" \ +./scripts/conformance.sh ``` -See [Dolt Backend for Beads](/architecture/dolt) for versioned storage and -[Configuration](/reference/configuration) for settings shared across storage -paths. +Adding a new backend is one declarative profile entry in `test/conformance/profiles.go` plus a store-factory arm; both tiers pick it up automatically. + +See [Dolt Backend for Beads](/architecture/dolt) for the default backend, [Configuration](/reference/configuration) for configuration, and [Troubleshooting](/reference/troubleshooting) if a connection misbehaves. diff --git a/docs/cli-reference/doctor.md b/docs/cli-reference/doctor.md index 64fc5abf1..be6a38e97 100644 --- a/docs/cli-reference/doctor.md +++ b/docs/cli-reference/doctor.md @@ -22,13 +22,6 @@ This command checks: - .beads/.gitignore up to date - Metadata.json version tracking (LastBdVersion field) -Storage Availability: - Full diagnostics, --perf, --deep, --server, --migration, and - --check=validate currently require Dolt server mode. Embedded Dolt and - SQLite support --check=artifacts, --check=conventions, and - --check=pollution. --check-health has a limited hook-health fallback. - Unsupported combinations return a notice without changing storage. - Performance Mode (--perf): Run performance diagnostics on your database: - Times key operations (bd ready, bd list, bd show, etc.) @@ -68,12 +61,18 @@ Server Mode (--server): - Schema compatible: Can query beads tables? - Connection pool: Pool health metrics -Legacy Dolt Migration Validation Mode (--migration): - Retained for older SQLite-to-Dolt migration workflows and available only in - Dolt server mode. It does not inspect or migrate a current SQLite workspace, - and it is not the migration path for removed PostgreSQL or MySQL backends. - Use the storage-backends migration guide for current backend moves. Combine - with --json for machine-parseable diagnostic output. +Migration Validation Mode (--migration): + Run Dolt migration validation checks with machine-parseable output. + Use --migration=pre before migration to verify readiness: + - JSONL file exists and is valid (parseable, no corruption) + - All JSONL issues are present in SQLite (or explains discrepancies) + - No blocking issues prevent migration + Use --migration=post after migration to verify completion: + - Dolt database exists and is healthy + - All issues from JSONL are present in Dolt + - No data was lost during migration + - Dolt database has no locks or uncommitted changes + Combine with --json for machine-parseable output for automation. Agent Mode (--agent): Output diagnostics designed for AI agent consumption. Instead of terse @@ -120,9 +119,9 @@ Examples: bd doctor --check=validate --fix # Auto-fix data-integrity issues bd doctor --deep # Full graph integrity validation bd doctor --server # Dolt server mode health checks - bd doctor --migration=pre # Legacy Dolt-server migration diagnostic - bd doctor --migration=post # Legacy Dolt-server completion diagnostic - bd doctor --migration=pre --json # Machine-parseable legacy diagnostic + bd doctor --migration=pre # Validate readiness for Dolt migration + bd doctor --migration=post # Validate Dolt migration completed + bd doctor --migration=pre --json # Machine-parseable migration validation ``` bd doctor [path] [flags] @@ -140,7 +139,7 @@ bd doctor [path] [flags] --fix Automatically fix issues where possible --fix-child-parent Remove childβ†’parent dependencies (opt-in) -i, --interactive Confirm each fix individually - --migration string Run legacy Dolt-server migration diagnostics: 'pre' or 'post' + --migration string Run Dolt migration validation: 'pre' (before migration) or 'post' (after migration) --orchestrator Running in orchestrator multi-workspace mode (routes.jsonl is expected, higher duplicate tolerance) --orchestrator-duplicates-threshold int Duplicate tolerance threshold for orchestrator mode (wisps are ephemeral) (default 1000) -o, --output string Export diagnostics to JSON file diff --git a/docs/cli-reference/init.md b/docs/cli-reference/init.md index b68d6e20a..23a9f7d3b 100644 --- a/docs/cli-reference/init.md +++ b/docs/cli-reference/init.md @@ -11,7 +11,7 @@ Initialize bd in the current directory by creating a .beads/ directory and its storage (a Dolt database by default). Optionally specify a custom issue prefix. Dolt is the default backend and the only one with version control (history, -branching, sync). Select the embedded SQLite alternative with --backend=sqlite; +branching, sync). Select an alternative with --backend=<postgres|mysql|sqlite>; see docs/architecture/storage-backends.md for the trade-offs and setup. Use --database to specify an existing server database name, overriding the @@ -52,7 +52,7 @@ bd init [flags] --agents-file string Custom filename for agent instructions (default: AGENTS.md) --agents-profile string AGENTS.md profile: 'minimal' (default, pointer to bd prime) or 'full' (complete command reference) --agents-template string Path to custom AGENTS.md template (overrides embedded default) - --backend string Storage backend: dolt (default) or sqlite. See docs/architecture/storage-backends.md. + --backend string Storage backend: dolt (default), postgres, mysql, or sqlite. See docs/architecture/storage-backends.md. --contributor Run OSS contributor setup wizard --database string Use existing server database name (overrides prefix-based naming) --debug Run the managed Dolt sql-server with --loglevel=debug and CPU profiling (--prof cpu). Persisted to config.yaml as dolt.debug. No effect on externally-managed servers. @@ -62,7 +62,11 @@ bd init [flags] --force Deprecated alias for --reinit-local. Bypasses only the LOCAL data-safety guard; does NOT authorize remote divergence (see 'bd help init-safety'). --from-jsonl Import issues from configured import.path; refuses remote history unless --discard-remote authorizes replacement --init-if-missing If the workspace is already initialized, skip init and exit 0 instead of failing (idempotent init for scaffolds) + --mysql-database string MySQL database for this workspace (with --backend=mysql; MySQL's isolation unit) + --mysql-url string MySQL server DSN (with --backend=mysql), e.g. user:pass@tcp(host:3306)/ . A password may be included for init but is never persisted; set BEADS_MYSQL_PASSWORD for later commands. Falls back to BEADS_MYSQL_URL. --non-interactive Skip all interactive prompts (auto-detected in CI or non-TTY environments) + --pg-schema string Postgres schema for this workspace's tables (with --backend=postgres; provides search_path isolation) + --pg-url string Postgres connection URL (with --backend=postgres). A password may be included for init but is never persisted; set BEADS_PG_PASSWORD for later commands. Falls back to BEADS_POSTGRES_URL. -p, --prefix string Issue prefix (default: current directory name) --proxied-server [EXPERIMENTAL] Use a per-workspace proxied dolt sql-server (proxy + child dolt) rooted at .beads/dolt --proxied-server-config-path string [EXPERIMENTAL] Absolute path to an existing dolt sql-server YAML config (proxied-server mode only). When set, bd uses this file instead of auto-generating one. Relative paths are rejected. diff --git a/docs/core-concepts/index.md b/docs/core-concepts/index.md index fd93a46c0..9568588ff 100644 --- a/docs/core-concepts/index.md +++ b/docs/core-concepts/index.md @@ -112,12 +112,10 @@ flowchart LR ## Sync β€” how work moves between machines -Dolt workspaces store issue data in -[Dolt](https://github.com/dolthub/dolt), a version-controlled SQL database. -Every write auto-commits to Dolt history; sync is native push/pull, -piggybacking on your existing git remote under a separate ref β€” no additional -sync server to run. SQLite workspaces do not provide this history or remote -sync path. +Beads stores everything in [Dolt](https://github.com/dolthub/dolt), a +version-controlled SQL database. Every write auto-commits to Dolt history; +sync is native push/pull, piggybacking on your existing git remote under a +separate ref β€” no server to run. ```mermaid flowchart LR @@ -145,13 +143,12 @@ is not the database, not the sync protocol, and not a backup. The full model | Mode | Command | Data lives at | Writers | |------|---------|---------------|---------| -| **Embedded Dolt** (default) | `bd init` | `.beads/embeddeddolt/` | one (file-locked) | -| **Dolt server** | `bd init --server` | managed local: `.beads/dolt/`; otherwise server-managed | many concurrent | -| **SQLite** | `bd init --backend=sqlite` | `.beads/beads.db` by default | serialized writes | +| **Embedded** (default) | `bd init` | `.beads/embeddeddolt/` | one (file-locked) | +| **Server** | `bd init --server` | `.beads/dolt/` | many concurrent | Embedded runs Dolt in-process and is right for almost everyone; server mode -connects to a `dolt sql-server` for multi-writer setups. SQLite provides a -small, server-free option without Dolt's history or sync. See +connects to an external `dolt sql-server` for multi-writer setups. Other SQL +backends exist without Dolt's history β€” see [Storage Backends](/architecture/storage-backends) and the [architecture overview](/architecture/index). diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index e83df4097..caaf03bb3 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -71,11 +71,9 @@ The wizard will: - Install git hooks (skip with `--skip-hooks`) Notes: -- Dolt is the default storage backend. Embedded data is stored in `.beads/embeddeddolt/`. +- Dolt is the default (and only) storage backend. Data is stored in `.beads/embeddeddolt/`. - By default, Dolt runs in **embedded mode** (in-process, no server needed). - For multi-writer setups, use `bd init --server` to connect to a `dolt sql-server` instead. -- For a small server-free workspace without Dolt history or sync, use - `bd init --backend=sqlite`. - To import issues from an older installation, run `bd init --from-jsonl`. ### Role configuration diff --git a/docs/multi-agent/federation.md b/docs/multi-agent/federation.md index a696ab603..107abb369 100644 --- a/docs/multi-agent/federation.md +++ b/docs/multi-agent/federation.md @@ -19,8 +19,7 @@ data between independent teams or locations. Key benefits: ## Prerequisites -1. **Dolt backend**: Federation requires embedded or server-mode Dolt; SQLite - does not provide the commit graph or remotes federation needs +1. **Dolt backend**: Federation requires the Dolt storage backend (the only supported backend) ## Configuration diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index e416a0431..eab6620ad 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -5,26 +5,18 @@ description: Complete reference for bd configuration across config.yaml and data Complete configuration reference for beads. -Last reviewed: 2026-07-16 +Last reviewed: 2026-07-10 Freshness source: `cmd/bd/main.go`, `cmd/bd/config.go`, and `internal/configfile/`. beads has two complementary configuration systems: 1. **Tool-level configuration** (YAML, managed by [Viper](https://github.com/spf13/viper)) β€” startup flags and tool behavior, stored in `config.yaml` files. These are user preferences: output format, auto-commit behavior, CLI ergonomics. -2. **Project-level configuration** (managed by `bd config`) β€” integration credentials, status maps, and project-specific settings, stored in the active storage database. Some keys are routed to `config.yaml` instead (see [YAML-only keys](#yaml-only-keys-startup-settings) below). +2. **Project-level configuration** (managed by `bd config`) β€” integration credentials, status maps, and project-specific settings, stored in the Dolt database. Some keys are routed to `config.yaml` instead (see [YAML-only keys](#yaml-only-keys-startup-settings) below). -The split is deliberate: tool settings are user-specific, while project config -travels with the active database. In a Dolt workspace it is team-shared when -you run `bd dolt push`; SQLite has no built-in cross-machine sync. Secrets are -refused in either database β€” see [Security](#security-where-secrets-live). +The split is deliberate: tool settings are user-specific; project config is team-shared and travels with the database when you run `bd dolt push`. That is also why secrets are refused in the database β€” see [Security](#security-where-secrets-live). -Supported storage paths are embedded Dolt (the default), Dolt server, and -SQLite. Embedded Dolt stores data at `.beads/embeddeddolt/`; managed repo-local -server mode (`bd init --server`) uses `.beads/dolt/`, while shared and external -servers manage their own data locations. SQLite uses `.beads/beads.db` by -default. See -[Storage Backends](/architecture/storage-backends). +Dolt is the only storage backend. Embedded mode (the default) stores data at `.beads/embeddeddolt/`; server mode (`bd init --server` or `BEADS_DOLT_SERVER_MODE=1`) uses `.beads/dolt/`. See [Dolt architecture](/architecture/dolt). ## Configuration Locations @@ -46,9 +38,7 @@ For Viper-managed (YAML) keys, highest to lowest: 3. **`config.yaml`** files (in the order listed above) 4. **Built-in defaults** -Project-level keys written via `bd config set` (Jira, Linear, GitHub, status -maps, etc.) live in the active storage database. They are read at command time -and have no env var override. +Project-level keys written via `bd config set` (Jira, Linear, GitHub, status maps, etc.) live in the Dolt database. They are read at command time and have no env var override. When a config.yaml value or environment variable shadows a database key, `bd config list` prints an override warning, and `bd config show` reports the source of every effective key. @@ -82,10 +72,7 @@ bd config validate bd config unset jira.url ``` -`bd config set` automatically routes the write to the right location: keys in -the YAML namespace (see below) are written to the project `config.yaml`; -everything else is written to the active storage database. `beads.role` is -stored in git config. +`bd config set` automatically routes the write to the right location: keys in the YAML namespace (see below) are written to the project `config.yaml`; everything else is written to the Dolt database. `beads.role` is stored in git config. Unrecognized keys produce a warning with a did-you-mean suggestion; use the `custom.*` namespace for user-defined keys. @@ -172,22 +159,17 @@ bd config set export.auto true bd config set export.git-add true ``` -For Dolt workspaces, use `bd dolt push` / `bd dolt pull` for sync and -`bd backup` for restorable database backups. SQLite has no native remote or -Dolt backup path; `bd export` remains available for issue-level portability. +Use `bd dolt push` / `bd dolt pull` for sync and `bd backup` for restorable +database backups. -Routing note: `output.title-length` and `agents.file` are functionally -tool-level settings, but `bd config set` writes them to the active storage -database. They are typically read from `config.yaml` when set there directly. +Routing note: `output.title-length` and `agents.file` are functionally tool-level settings, but `bd config set` writes them to the Dolt database. They are typically read from `config.yaml` when set there directly. `bd config show` is the source of truth for what's currently effective on your machine, including provenance. ## Dolt History, Backup, and Push -For Dolt workspaces, three post-write behaviors run after each successful write -command, in this order: auto-commit, auto-backup, auto-push. SQLite commits its -SQL transaction and skips this Dolt-specific maintenance tail. +Three post-write behaviors run after each successful write command, in this order: auto-commit, auto-backup, auto-push. ### Auto-commit: SQL Commits vs Dolt Commits @@ -277,8 +259,7 @@ export BEADS_ACTOR="my-github-handle" ## Project-Level Settings (Database) -These are written to the active storage database by `bd config set` and have no -env var override. Common namespaces: +These are written to the Dolt database by `bd config set` and have no env var override. Common namespaces: | Namespace | Purpose | |---|---| @@ -372,10 +353,7 @@ bd config set max_hash_length "8" # Upper bound (default 8) ## Sync and Federation -Native sync and federation require Dolt. Dolt workspaces use remotes -(`bd dolt push` / `bd dolt pull`) with cell-level merge and `bd backup` for -restorable database backups. SQLite has no native remote; use `bd export` for -issue portability. +Beads syncs exclusively through Dolt remotes (`bd dolt push` / `bd dolt pull`) with cell-level merge. Use `bd export` for issue portability and `bd backup` for restorable database backups. Federation settings live in `config.yaml`: @@ -488,14 +466,9 @@ Integration secrets follow tracker-specific conventions: `LINEAR_API_KEY`, `GITH ## Security: Where Secrets Live -- Tokens and API keys are never stored in the active storage database. Dolt - database config can be pushed to remotes, which would expose secrets and trip - GitHub secret scanning; SQLite follows the same safe rule. `bd config set` - routes secret keys to the local `config.yaml` instead. +- Tokens and API keys are never stored in the Dolt database β€” database config is pushed to remotes, which would expose secrets and trip GitHub secret scanning. `bd config set` routes secret keys to the local `config.yaml` instead. - Writing a secret to a git-tracked `config.yaml` is refused unless you pass `--force-git-tracked`; environment variables are the safer default. -- `bd init` writes a `.beads/.gitignore` that keeps the database storage - (`embeddeddolt/`, `dolt/`, and SQLite database files), runtime files, push - state, and the federation credential key out of git. +- `bd init` writes a `.beads/.gitignore` that keeps the database directories (`embeddeddolt/`, `dolt/`), runtime files, push state, and the federation credential key out of git. ## Example `.beads/config.yaml` diff --git a/docs/reference/faq.md b/docs/reference/faq.md index 6ef0b2445..670145e6e 100644 --- a/docs/reference/faq.md +++ b/docs/reference/faq.md @@ -7,12 +7,7 @@ description: Common questions about beads and how to use it effectively ### What is beads? -Beads (`bd`) is a lightweight, local-first issue tracker designed for AI coding -agents. It uses version-controlled Dolt storage by default and also supports -SQLite for small server-free workspaces. It provides dependency-aware task -management, with Dolt-backed sync across machines so agents and humans can -collaborate from the same task graph. See -[Core Concepts](/core-concepts/index) for the full model. +Beads (`bd`) is a lightweight, Dolt-backed issue tracker designed for AI coding agents. It provides dependency-aware task management with built-in sync across machines, so agents and humans can collaborate from the same task graph. See [Core Concepts](/core-concepts/index) for the full model. ### Why beads instead of GitHub Issues or Jira? @@ -35,9 +30,9 @@ Taskwarrior is excellent for personal task management, but beads is built for AI - **Agent semantics**: the `discovered-from` dependency type, `bd ready` for queue management - **JSON-first design**: every command has `--json` output -- **Built-in Dolt sync**: version-controlled storage with native push/pull, no separate sync server to run -- **Cell-level Dolt merge**: concurrent changes merge automatically at the field level -- **SQL storage**: full SQL queries against Dolt, with SQLite available for smaller local workspaces +- **Built-in sync**: version-controlled storage with native push/pull, no separate sync server to run +- **Cell-level merge**: concurrent changes merge automatically at the field level +- **SQL database**: full SQL queries against the Dolt database ### Can I use beads without AI agents? @@ -49,16 +44,11 @@ Nothing specific β€” it's a metaphor for linked work items, like beads on a stri ### Is beads production-ready? -Beads is a 1.x product used in production for AI-assisted development. The core -functionality β€” create, update, dependencies, ready work, and Dolt-backed sync β€” -is stable, and releases follow semantic versioning. `bd export` produces -human-readable JSONL for issue-level portability, but JSONL is not a complete -database backup. Dolt workspaces can use a Dolt remote or `bd backup`; protect a -SQLite workspace with SQLite-aware or offline file-backup tooling. +Beads is a 1.x product used in production for AI-assisted development. The core functionality β€” create, update, dependencies, ready work, Dolt-backed sync β€” is stable, and releases follow semantic versioning. Data stays portable: `bd export` produces human-readable JSONL, and `bd backup` pushes Dolt-native backups. As with any tracker holding work you care about, keep normal backup hygiene (a Dolt remote or a `bd backup` destination). ## Architecture -### Why is Dolt the default instead of plain SQLite or flat files? +### Why Dolt instead of plain SQLite or flat files? Dolt is a version-controlled SQL database β€” git semantics at the database level: @@ -68,10 +58,7 @@ Dolt is a version-controlled SQL database β€” git semantics at the database leve - **Multi-writer**: server mode supports concurrent agents - **Portable**: `bd export` produces JSONL for migration and interoperability -SQLite remains available when a small server-free database matters more than -history and sync. See [Dolt architecture](/architecture/dolt) for the detailed -analysis and [Storage backends](/architecture/storage-backends) for the -supported paths. +See [Dolt architecture](/architecture/dolt) for the detailed analysis and [Storage backends](/architecture/storage-backends) for the alternatives. ### Why hash-based IDs instead of sequential? @@ -105,16 +92,7 @@ The parent hash keeps the namespace unique across epics, the child numbers stay The default `bd init` uses **embedded mode**: Dolt runs in-process inside `bd`, data lives at `.beads/embeddeddolt/`, and there is no server, port, or PID file to manage. This is the right mode for solo work, CI/CD, and single-agent setups. -**Dolt server mode** (`bd init --server`) connects to a running -`dolt sql-server`. Managed repo-local mode stores data at `.beads/dolt/`; -shared and external servers manage data at the server. Switch to it when -multiple processes need concurrent write access to the same database β€” for -example several agents on one machine. See -[Dolt architecture](/architecture/dolt) for setup and migration between modes. - -SQLite is a separate server-free storage path selected with -`bd init --backend=sqlite`; it is neither embedded-Dolt mode nor Dolt-server -mode and does not provide Dolt history or sync. +**Server mode** (`bd init --server`) connects to a running `dolt sql-server` and stores data at `.beads/dolt/`. Switch to it when multiple processes need concurrent write access to the same database β€” for example several agents on one machine. See [Dolt architecture](/architecture/dolt) for setup and migration between modes. ## Usage @@ -127,10 +105,7 @@ bd init # Humans: interactive, prompts for git hooks bd init --quiet # Agents: non-interactive, auto-installs hooks ``` -For an existing Dolt-backed project, clone and run `bd init`; it creates the -local Dolt database and pulls from the configured remote. For a new project, -`bd init` chooses embedded Dolt by default; use `bd init --backend=sqlite` for -the smaller server-free SQLite path. +For an existing project, clone and run `bd init`; it creates the Dolt database and pulls from the configured remote. For a new project, run `bd init`, then commit the `.beads/` directory. ### How do I sync issues across machines? @@ -139,22 +114,11 @@ bd dolt push # Push changes to the Dolt remote bd dolt pull # Pull changes from the Dolt remote ``` -These commands apply to Dolt workspaces. `bd init` auto-configures your git -origin as the Dolt remote when present; use `bd init --remote ` for an -explicit remote. SQLite has no native Beads sync; choose Dolt when the same -issue graph must move across machines. See [Sync Setup](/getting-started/sync-setup). +`bd init` auto-configures your git origin as the Dolt remote when present; use `bd init --remote ` for an explicit remote. See [Sync Setup](/getting-started/sync-setup). ### Do I need to run export/import manually? -Not for normal operation. Dolt writes are committed to Dolt history, and -`bd dolt push` / `bd dolt pull` handle sync. SQLite writes go directly to the -configured SQLite file and have no Dolt history or native Beads sync. - -For either backend, `bd export` and `bd import` are issue-level portability and -migration tools. `.beads/issues.jsonl` is a passive interchange file, never the -database or a complete backup. Dolt workspaces can use `bd backup`; protect -SQLite with SQLite-aware or offline file-backup tooling. See -[Sync Concepts](/core-concepts/sync-concepts) for the full Dolt model. +No. All writes go directly to the Dolt database and are committed to Dolt history automatically; `bd dolt push` / `bd dolt pull` handle sync. `bd export` exists for portability and interchange β€” `.beads/issues.jsonl` is a passive export, never the database. For backups, use `bd backup init ` / `bd backup sync` / `bd backup restore`. See [Sync Concepts](/core-concepts/sync-concepts) for the full model and the sync patterns to avoid. ### What if my database feels stale after a colleague pushes changes? @@ -163,9 +127,7 @@ bd dolt pull # Fetch and merge updates from the Dolt remote bd ready # Shows fresh data ``` -This is a Dolt-workspace workflow. For federation setups, `bd federation sync` -syncs with all configured peers. SQLite workspaces do not pull colleague -changes. See [Federation](/multi-agent/federation). +For federation setups, `bd federation sync` syncs with all configured peers. See [Federation](/multi-agent/federation). ### How do I handle merge conflicts? @@ -205,11 +167,7 @@ In orchestrated workflows an orchestrator usually assigns work (`bd assign`); ag ### Does beads work offline? -Yes β€” beads is offline-first. Queries run against local storage and normal -commands do not need the network. Dolt workspaces sync later with -`bd dolt push` / `bd dolt pull`; SQLite remains local and has no native Beads -sync. That makes both paths suitable for planes, unstable connections, -air-gapped environments, and privacy-sensitive projects. +Yes β€” beads is offline-first. All queries run against the local Dolt database, no command needs the network, and sync happens via `bd dolt push` / `bd dolt pull` when you're online. That makes it suitable for planes, unstable connections, air-gapped environments, and privacy-sensitive projects. ### How do I use beads in CI/CD? @@ -311,11 +269,7 @@ Sure β€” beads is just an issue tracker. Writing projects (chapters as issues, o ### What dependencies does beads have? -Beads is a single static binary with no runtime dependencies β€” the Dolt engine -and SQLite driver are embedded in-process. No PostgreSQL server, generic MySQL -server, Redis, Docker, or node_modules are required. The standalone `dolt` CLI -is only needed if you run Dolt server mode, and git is only needed to version -your project code. See [Installation](/getting-started/installation). +Beads is a single static binary with no runtime dependencies β€” the Dolt engine is embedded in-process. No PostgreSQL, no Redis, no Docker, no node_modules. The standalone `dolt` CLI is only needed if you run server mode, and git is only needed to version your project code. See [Installation](/getting-started/installation). ### Can I query or extend the database directly? diff --git a/examples/library-usage/README.md b/examples/library-usage/README.md index 6f9774950..9eac33788 100644 --- a/examples/library-usage/README.md +++ b/examples/library-usage/README.md @@ -35,9 +35,9 @@ import ( func main() { ctx := context.Background() - // Find the workspace and open its configured Dolt or SQLite implementation - beadsDir := beads.FindBeadsDir() - store, err := beads.OpenBestAvailable(ctx, beadsDir) + // Find and open database + dbPath := beads.FindDatabasePath() + store, err := beads.Open(ctx, dbPath) if err != nil { log.Fatal(err) } @@ -145,8 +145,8 @@ type VCStorage struct { beads beads.Storage } -func NewVCStorage(ctx context.Context, beadsDir string) (*VCStorage, error) { - store, err := beads.OpenBestAvailable(ctx, beadsDir) +func NewVCStorage(ctx context.Context, dbPath string) (*VCStorage, error) { + store, err := beads.Open(ctx, dbPath) if err != nil { return nil, err } diff --git a/examples/library-usage/main.go b/examples/library-usage/main.go index 087d4bc03..3596776d7 100644 --- a/examples/library-usage/main.go +++ b/examples/library-usage/main.go @@ -16,16 +16,16 @@ import ( func main() { ctx := context.Background() - // Find the Beads workspace so backend selection can follow metadata.json. - beadsDir := beads.FindBeadsDir() - if beadsDir == "" { - log.Fatal("No Beads workspace found. Run 'bd init' first.") + // Find the Beads database (looks for .beads/*.db in current/parent dirs) + dbPath := beads.FindDatabasePath() + if dbPath == "" { + log.Fatal("No Beads database found. Run 'bd init' first.") } - fmt.Printf("Using workspace: %s\n\n", beadsDir) + fmt.Printf("Using database: %s\n\n", dbPath) - // Open the configured Dolt or SQLite implementation. - store, err := beads.OpenBestAvailable(ctx, beadsDir) + // Open the database + store, err := beads.Open(ctx, dbPath) if err != nil { log.Fatalf("Failed to open storage: %v", err) } diff --git a/examples/library-usage/main_test.go b/examples/library-usage/main_test.go index f6780eeee..92953c6ff 100644 --- a/examples/library-usage/main_test.go +++ b/examples/library-usage/main_test.go @@ -18,17 +18,11 @@ func TestExampleCompiles(t *testing.T) { } defer os.RemoveAll(tmpDir) - beadsDir := filepath.Join(tmpDir, ".beads") - if err := os.MkdirAll(beadsDir, 0o750); err != nil { - t.Fatalf("Failed to create .beads dir: %v", err) - } - if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"sqlite","sqlite_path":"beads.db"}`), 0o600); err != nil { - t.Fatalf("Failed to create metadata.json: %v", err) - } + dbPath := filepath.Join(tmpDir, "dolt") // Open storage ctx := context.Background() - store, err := beads.OpenBestAvailable(ctx, beadsDir) + store, err := beads.Open(ctx, dbPath) if err != nil { t.Fatalf("Failed to open storage: %v", err) } diff --git a/go.mod b/go.mod index 7fc873288..abfdb204a 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/dolthub/driver/v2 v2.2.0 github.com/dolthub/eventkit v0.0.0-20260611184414-99f5693e696a github.com/go-sql-driver/mysql v1.10.0 + github.com/jackc/pgx/v5 v5.10.0 github.com/olebedev/when v1.1.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 @@ -44,6 +45,9 @@ require ( github.com/cockroachdb/apd/v3 v3.2.3 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/moby/moby/api v1.54.1 // indirect github.com/moby/moby/client v0.4.0 // indirect diff --git a/go.sum b/go.sum index 4dbbe7535..29901bb04 100644 --- a/go.sum +++ b/go.sum @@ -724,6 +724,7 @@ github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bY github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= @@ -734,6 +735,8 @@ github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= @@ -744,10 +747,14 @@ github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9 github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= diff --git a/internal/beads/beads.go b/internal/beads/beads.go index dc08adc22..baf010799 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -421,37 +421,13 @@ func findLocalBeadsDir() string { } // findDatabaseInBeadsDir searches for a database within a .beads directory. -// Checks metadata.json for the selected implementation's database path. For -// server mode and SQLite, no local path needs to exist yet: authoritative -// metadata is enough to route the caller without falling through to Dolt. -// Embedded Dolt checks both embeddeddolt/ and the legacy dolt/ path. Returns -// empty string if no database is configured or found. +// Checks metadata.json for the Dolt database path. For server mode, no local +// directory is required. For embedded mode, checks both the embeddeddolt/ +// directory (where the embedded engine stores data) and the legacy dolt/ path. +// Returns empty string if no database is found. func findDatabaseInBeadsDir(beadsDir string, _ bool) string { // Check for metadata.json first (single source of truth) - cfg, err := configfile.Load(beadsDir) - if err != nil { - // A present but unreadable/invalid metadata file is authoritative. Do - // not bypass it by discovering a leftover local Dolt directory; callers - // that need the detailed error should use OpenBestAvailable. - return "" - } - if cfg != nil { - if !configfile.IsSupportedBackend(cfg.Backend) { - // Unknown and removed backend markers are not Dolt aliases. Let the - // storage-selection layer report the metadata error without routing - // through a leftover local Dolt directory. - return "" - } - if cfg.GetBackend() == configfile.BackendSQLite { - dbPath := cfg.GetSQLitePath() - if dbPath == "" { - dbPath = "beads.db" - } - if !filepath.IsAbs(dbPath) { - dbPath = filepath.Join(beadsDir, dbPath) - } - return dbPath - } + if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil { // For Dolt server mode, database is on the server - no local directory required if cfg.IsDoltServerMode() { return cfg.DatabasePath(beadsDir) @@ -515,7 +491,6 @@ func FindDatabasePath() string { // BEADS_DIR is set but no database found - this is OK for --no-db mode // Return empty string and let the caller handle it - return "" } // 2. Check BEADS_DB environment variable (deprecated but still supported) diff --git a/internal/beads/sqlite_database_path_test.go b/internal/beads/sqlite_database_path_test.go deleted file mode 100644 index 2939eee1a..000000000 --- a/internal/beads/sqlite_database_path_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package beads - -import ( - "os" - "path/filepath" - "testing" -) - -func TestFindDatabasePathSQLiteMetadataOutsideCWD(t *testing.T) { - tests := []struct { - name string - metadata string - relativePath string - }{ - { - name: "default path", - metadata: `{"backend":"sqlite","sqlite_path":"beads.db"}`, - relativePath: "beads.db", - }, - { - name: "custom relative path", - metadata: `{"backend":"sqlite","sqlite_path":"data/custom.db"}`, - relativePath: filepath.Join("data", "custom.db"), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - root := t.TempDir() - cwd := filepath.Join(root, "working-directory") - beadsDir := filepath.Join(root, "external-workspace", ".beads") - if err := os.MkdirAll(cwd, 0o750); err != nil { - t.Fatalf("create working directory: %v", err) - } - if err := os.MkdirAll(beadsDir, 0o750); err != nil { - t.Fatalf("create beads directory: %v", err) - } - if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(tt.metadata), 0o600); err != nil { - t.Fatalf("write metadata.json: %v", err) - } - - t.Chdir(cwd) - t.Setenv("BEADS_DIR", beadsDir) - t.Setenv("BEADS_DB", "") - t.Setenv("BD_DB", "") - - want := filepath.Join(beadsDir, tt.relativePath) - if _, err := os.Stat(want); !os.IsNotExist(err) { - t.Fatalf("SQLite database unexpectedly exists before discovery: %v", err) - } - - if got := FindDatabasePath(); got != want { - t.Errorf("FindDatabasePath() = %q, want %q", got, want) - } - if _, err := os.Stat(want); !os.IsNotExist(err) { - t.Fatalf("FindDatabasePath created SQLite database path %q: %v", want, err) - } - }) - } -} - -func TestFindDatabasePathDoesNotBypassCorruptMetadata(t *testing.T) { - root := t.TempDir() - beadsDir := filepath.Join(root, "selected", ".beads") - leftoverDolt := filepath.Join(beadsDir, "embeddeddolt") - if err := os.MkdirAll(leftoverDolt, 0o750); err != nil { - t.Fatalf("create leftover Dolt directory: %v", err) - } - if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte("{"), 0o600); err != nil { - t.Fatalf("write corrupt metadata.json: %v", err) - } - ambiguousCWD := filepath.Join(root, "ambient") - if err := os.MkdirAll(filepath.Join(ambiguousCWD, ".beads", "embeddeddolt"), 0o750); err != nil { - t.Fatalf("create ambient Dolt workspace: %v", err) - } - t.Chdir(ambiguousCWD) - - t.Setenv("BEADS_DIR", beadsDir) - t.Setenv("BEADS_DB", "") - t.Setenv("BD_DB", "") - - if got := FindDatabasePath(); got != "" { - t.Fatalf("FindDatabasePath() = %q with corrupt metadata, want empty path instead of leftover Dolt", got) - } -} diff --git a/internal/configfile/backend_messages.go b/internal/configfile/backend_messages.go deleted file mode 100644 index db026a5a1..000000000 --- a/internal/configfile/backend_messages.go +++ /dev/null @@ -1,38 +0,0 @@ -package configfile - -import "fmt" - -// This file is the single source of truth for the user-facing removed-backend -// and unknown-backend messages. cmd/bd, the public beads package, and the Dolt -// store must all fail closed with the same text, so the wording is deliberately -// centralized here. The framing of these messages is pending a separate -// maintainer decision β€” reword them here, in one place, once that lands. - -// RemovedBackendRationale explains why direct PostgreSQL/MySQL support was -// removed. Shortened site-specific messages (e.g. bd init flag guidance) -// compose with this clause directly. -const RemovedBackendRationale = "direct support for general-purpose server databases was rolled back to keep Beads simple and resource-light" - -// BackendNotOpenedGuarantee is the fail-closed data-safety guarantee included -// in backend rejection errors: refusing a workspace never opens, creates, or -// modifies its storage. -const BackendNotOpenedGuarantee = "no storage database was opened or modified" - -// RemovedBackendDetail returns the shared body of every removed-backend error: -// the rationale, the untouched-data guarantee, and the migration path. Callers -// prepend a site-specific lead-in; RemovedBackendError carries the standard one. -func RemovedBackendDetail(backend string) string { - return fmt.Sprintf("%s; the configured %s database was not opened or modified; export it with a bd version that supports %s, then follow bd help init-safety to reinitialize with Dolt or SQLite and import the exported data", RemovedBackendRationale, backend, backend) -} - -// RemovedBackendError is the standard fail-closed error for metadata that -// selects a backend whose direct support was removed. -func RemovedBackendError(backend string) error { - return fmt.Errorf("storage backend %q is no longer supported: %s", backend, RemovedBackendDetail(backend)) -} - -// UnknownBackendError is the standard fail-closed error for metadata that -// names a backend this build does not recognize. -func UnknownBackendError(backend string) error { - return fmt.Errorf("storage backend %q in metadata.json is not recognized or supported; %s; supported backends are %q and %q; fix or restore metadata.json and retry", backend, BackendNotOpenedGuarantee, BackendDolt, BackendSQLite) -} diff --git a/internal/configfile/configfile.go b/internal/configfile/configfile.go index d0c4351e8..712d2a4cf 100644 --- a/internal/configfile/configfile.go +++ b/internal/configfile/configfile.go @@ -17,7 +17,7 @@ const ConfigFileName = "metadata.json" type Config struct { Database string `json:"database"` - Backend string `json:"backend,omitempty"` // Storage backend: "dolt" (default) or "sqlite"; legacy "postgres"/"mysql" values are rejection tombstones. Read via GetBackend(). + Backend string `json:"backend,omitempty"` // Storage backend: "dolt" (default), "postgres", "mysql", or "sqlite". Read via GetBackend(). // Deletions configuration DeletionsRetentionDays int `json:"deletions_retention_days,omitempty"` // 0 means use default (3 days) @@ -36,13 +36,15 @@ type Config struct { DoltRemotesAPIPort int `json:"dolt_remotesapi_port,omitempty"` // Dolt remotesapi port for federation (default: 8080) // Note: Password should be set via BEADS_DOLT_PASSWORD env var for security - // Deprecated backend fields are retained only to round-trip metadata written by - // the short-lived PostgreSQL/MySQL implementations. They are not connection - // configuration for current builds; store selection rejects those backends. - PostgresDSN string `json:"postgres_dsn,omitempty"` - PostgresSchema string `json:"postgres_schema,omitempty"` - MySQLDSN string `json:"mysql_dsn,omitempty"` - MySQLDatabase string `json:"mysql_database,omitempty"` + // Postgres backend (backend="postgres"). Password is NEVER persisted; it + // comes from BEADS_PG_PASSWORD or BEADS_POSTGRES_URL. + PostgresDSN string `json:"postgres_dsn,omitempty"` // e.g. postgres://user@host:5432/db (no password) + PostgresSchema string `json:"postgres_schema,omitempty"` // per-workspace schema (search_path) + + // MySQL backend (backend="mysql"). Password is NEVER persisted; it comes from + // BEADS_MYSQL_PASSWORD or BEADS_MYSQL_URL. + MySQLDSN string `json:"mysql_dsn,omitempty"` // e.g. user@tcp(host:3306)/ (no password) + MySQLDatabase string `json:"mysql_database,omitempty"` // per-workspace database (MySQL's isolation unit) // SQLite backend (backend="sqlite"). File-based, embedded; no credentials. SQLitePath string `json:"sqlite_path,omitempty"` // database file, relative to the beads dir (default beads.db) @@ -229,9 +231,9 @@ type BackendCapabilities struct { SingleProcessOnly bool } -// CapabilitiesForBackend returns capabilities for a backend string. Embedded Dolt -// and SQLite are single-process-only; use Config.GetCapabilities() to account for -// Dolt server and proxied-server modes. +// CapabilitiesForBackend returns capabilities for a backend string. +// Dolt is the only supported backend. Returns SingleProcessOnly=true by default; +// use Config.GetCapabilities() to properly handle server mode. func CapabilitiesForBackend(_ string) BackendCapabilities { return BackendCapabilities{SingleProcessOnly: true} } @@ -247,18 +249,10 @@ func (c *Config) GetCapabilities() BackendCapabilities { return CapabilitiesForBackend(backend) } -// IsSupportedBackend reports whether backend selects an implementation shipped by -// beads. The empty value is the legacy/default spelling of Dolt. -func IsSupportedBackend(backend string) bool { - return backend == "" || backend == BackendDolt || backend == BackendSQLite -} - -// GetBackend returns the configured storage backend. PostgreSQL and MySQL remain -// recognizable here so workspaces created by earlier releases can fail loudly at -// store selection instead of silently falling back to an empty Dolt database. -// Empty and explicit Dolt retain the established Dolt behavior. GetBackend keeps -// the historical Dolt fallback for unknown values, so storage-selection callers -// must check IsSupportedBackend(c.Backend) before opening or creating storage. +// GetBackend returns the configured storage backend. Only the explicitly-allowlisted +// non-default backends ("postgres", "mysql") are honored; "", "dolt", and any legacy +// or unknown value resolve to dolt so the default path stays byte-identical and a +// typo fails safe to Dolt. func (c *Config) GetBackend() string { if c != nil { switch c.Backend { @@ -282,6 +276,82 @@ func (c *Config) GetSQLitePath() string { return c.SQLitePath } +// GetPostgresDSN returns the base Postgres connection string: BEADS_POSTGRES_URL (a +// full override that may carry a password) if set, else the persisted, +// password-free metadata postgres_dsn. It never merges a password β€” password +// resolution (BEADS_PG_PASSWORD_COMMAND, BEADS_PG_PASSWORD, the credentials file) +// and placement happen at open time in the postgres backend, which owns the pgx +// parser this low-level package must not import (see the note at RedactPassword). +func (c *Config) GetPostgresDSN() string { + if u := os.Getenv("BEADS_POSTGRES_URL"); u != "" { + return u + } + if c == nil { + return "" + } + return c.PostgresDSN +} + +// GetPostgresPasswordCommand returns the credential command that resolves the +// Postgres password: BEADS_PG_PASSWORD_COMMAND. Empty means no command β€” the static +// BEADS_PG_PASSWORD / credentials-file path applies. The command's stdout is a +// password (a bare token or a {token,expires_in} envelope); it is run at open time, +// out-ranking the static password so a rotating secret is never shadowed by a stale +// env value. It is deliberately read from the environment only, NOT metadata.json: +// a metadata-sourced command is arbitrary code run on open, so persisting it waits +// on a workspace-trust gate. +func (c *Config) GetPostgresPasswordCommand() string { + return os.Getenv("BEADS_PG_PASSWORD_COMMAND") +} + +// GetPostgresSchema returns the per-workspace Postgres schema (search_path). +func (c *Config) GetPostgresSchema() string { + if c == nil { + return "" + } + return c.PostgresSchema +} + +// GetMySQLDSN returns the base MySQL server DSN: BEADS_MYSQL_URL (a full override +// that may carry a password) if set, else the persisted, password-free metadata +// mysql_dsn. It never merges a password β€” password resolution +// (BEADS_MYSQL_PASSWORD_COMMAND, BEADS_MYSQL_PASSWORD, the credentials file) and +// placement happen at open time in the mysql backend, which owns the go-sql-driver +// parser this low-level package must not import (see the note at RedactPassword). +func (c *Config) GetMySQLDSN() string { + if u := os.Getenv("BEADS_MYSQL_URL"); u != "" { + return u + } + if c == nil { + return "" + } + return c.MySQLDSN +} + +// GetMySQLPasswordCommand returns the credential command that resolves the MySQL +// password: BEADS_MYSQL_PASSWORD_COMMAND. Empty means no command β€” the static +// BEADS_MYSQL_PASSWORD / credentials-file path applies. The command's stdout is a +// password (a bare token or a {token,expires_in} envelope); it is run at open time, +// out-ranking the static password so a rotating secret is never shadowed by a stale +// env value. It is deliberately read from the environment only, NOT metadata.json: +// a metadata-sourced command is arbitrary code run on open, so persisting it waits +// on a workspace-trust gate. +func (c *Config) GetMySQLPasswordCommand() string { + return os.Getenv("BEADS_MYSQL_PASSWORD_COMMAND") +} + +// GetMySQLDatabase returns the per-workspace MySQL database (isolation unit). +func (c *Config) GetMySQLDatabase() string { + if c == nil { + return "" + } + return c.MySQLDatabase +} + +// Password redaction for persistence lives in pgdialect.RedactPassword, not here: +// it must fail closed (verify with pgx that no password survives) and therefore +// depends on the pgx parser, which this low-level config package must not import. + // Dolt mode constants const ( DoltModeEmbedded = "embedded" diff --git a/internal/configfile/configfile_test.go b/internal/configfile/configfile_test.go index 8bc1a725c..9f9bbed8e 100644 --- a/internal/configfile/configfile_test.go +++ b/internal/configfile/configfile_test.go @@ -629,10 +629,10 @@ func TestProxiedServerClientInfo_ResolvedPaths(t *testing.T) { }) } -// TestGetBackendAllowlist verifies the metadata-routing semantics: current backends -// and removed-backend tombstones remain recognizable, while empty or unknown values -// retain the historical Dolt fallback. Store selection rejects the tombstones before -// it can open an empty Dolt database. +// TestGetBackendAllowlist verifies the allowlist semantics: the SQL backends +// (postgres, mysql, sqlite) are honored; every other value (empty, legacy, +// genuinely unknown) falls back to Dolt. This is the guard behind backend selection +// β€” a typo in metadata.json must fail safe to Dolt, never to an unintended backend. func TestGetBackendAllowlist(t *testing.T) { fallsBackToDolt := []struct { name string @@ -662,29 +662,6 @@ func TestGetBackendAllowlist(t *testing.T) { } } -func TestSupportedBackendAllowlist(t *testing.T) { - tests := []struct { - name string - backend string - supported bool - }{ - {name: "implicit dolt", backend: "", supported: true}, - {name: "dolt", backend: BackendDolt, supported: true}, - {name: "sqlite", backend: BackendSQLite, supported: true}, - {name: "postgres", backend: BackendPostgres, supported: false}, - {name: "mysql", backend: BackendMySQL, supported: false}, - {name: "unknown", backend: "mystery", supported: false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := IsSupportedBackend(tt.backend); got != tt.supported { - t.Fatalf("IsSupportedBackend(%q) = %v, want %v", tt.backend, got, tt.supported) - } - }) - } -} - // TestDatabasePathAlwaysDolt tests that DatabasePath always returns the dolt path. func TestDatabasePathAlwaysDolt(t *testing.T) { beadsDir := "/home/user/project/.beads" diff --git a/internal/configfile/mysql_dsn_test.go b/internal/configfile/mysql_dsn_test.go new file mode 100644 index 000000000..7749a22f3 --- /dev/null +++ b/internal/configfile/mysql_dsn_test.go @@ -0,0 +1,37 @@ +package configfile + +import "testing" + +// GetMySQLDSN returns the BASE DSN and never merges a password: BEADS_MYSQL_PASSWORD +// is resolved and placed at open time in the mysql backend (which owns the +// go-sql-driver parser), so the persisted, password-free metadata DSN comes back +// unchanged here. +func TestGetMySQLDSNReturnsBaseUnmerged(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD", "secret") + c := &Config{MySQLDSN: "bts@tcp(127.0.0.1:3306)/"} + if got, want := c.GetMySQLDSN(), "bts@tcp(127.0.0.1:3306)/"; got != want { + t.Fatalf("GetMySQLDSN() = %q, want %q (must not merge the password)", got, want) + } +} + +// BEADS_MYSQL_URL (full override) takes precedence over the metadata DSN. +func TestGetMySQLDSNEnvURLPrecedence(t *testing.T) { + t.Setenv("BEADS_MYSQL_URL", "root:pw@tcp(host:3306)/db") + c := &Config{MySQLDSN: "other@tcp(127.0.0.1:3306)/"} + if got, want := c.GetMySQLDSN(), "root:pw@tcp(host:3306)/db"; got != want { + t.Fatalf("GetMySQLDSN() = %q, want %q", got, want) + } +} + +// GetMySQLPasswordCommand reads BEADS_MYSQL_PASSWORD_COMMAND from the environment +// only (metadata is deliberately excluded until a workspace-trust gate exists). +func TestGetMySQLPasswordCommand(t *testing.T) { + c := &Config{} + if got := c.GetMySQLPasswordCommand(); got != "" { + t.Fatalf("GetMySQLPasswordCommand() = %q, want empty when unset", got) + } + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "get-credential") + if got, want := c.GetMySQLPasswordCommand(), "get-credential"; got != want { + t.Fatalf("GetMySQLPasswordCommand() = %q, want %q", got, want) + } +} diff --git a/internal/configfile/postgres_dsn_test.go b/internal/configfile/postgres_dsn_test.go new file mode 100644 index 000000000..0ffbc57d3 --- /dev/null +++ b/internal/configfile/postgres_dsn_test.go @@ -0,0 +1,36 @@ +package configfile + +import "testing" + +// GetPostgresDSN returns the BASE DSN and never merges a password: BEADS_PG_PASSWORD +// is resolved and placed at open time in the postgres backend (which owns the pgx +// parser), so the persisted, password-free metadata DSN comes back unchanged here. +func TestGetPostgresDSNReturnsBaseUnmerged(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD", "secret") + c := &Config{PostgresDSN: "postgres://bts@127.0.0.1:5432/db"} + if got, want := c.GetPostgresDSN(), "postgres://bts@127.0.0.1:5432/db"; got != want { + t.Fatalf("GetPostgresDSN() = %q, want %q (must not merge the password)", got, want) + } +} + +// BEADS_POSTGRES_URL (full override) takes precedence over the metadata DSN. +func TestGetPostgresDSNEnvURLPrecedence(t *testing.T) { + t.Setenv("BEADS_POSTGRES_URL", "postgres://a:b@host:5432/db") + c := &Config{PostgresDSN: "postgres://other@127.0.0.1:5432/db"} + if got, want := c.GetPostgresDSN(), "postgres://a:b@host:5432/db"; got != want { + t.Fatalf("GetPostgresDSN() = %q, want %q", got, want) + } +} + +// GetPostgresPasswordCommand reads BEADS_PG_PASSWORD_COMMAND from the environment +// only (metadata is deliberately excluded until a workspace-trust gate exists). +func TestGetPostgresPasswordCommand(t *testing.T) { + c := &Config{} + if got := c.GetPostgresPasswordCommand(); got != "" { + t.Fatalf("GetPostgresPasswordCommand() = %q, want empty when unset", got) + } + t.Setenv("BEADS_PG_PASSWORD_COMMAND", "get-credential") + if got, want := c.GetPostgresPasswordCommand(), "get-credential"; got != want { + t.Fatalf("GetPostgresPasswordCommand() = %q, want %q", got, want) + } +} diff --git a/internal/storage/backend_marker.go b/internal/storage/backend_marker.go index 1fd674cdd..717b9614b 100644 --- a/internal/storage/backend_marker.go +++ b/internal/storage/backend_marker.go @@ -2,14 +2,15 @@ package storage // NonCommitGraphBackend is an affirmative *negative* marker: it is implemented // only by backends whose Commit/GetCurrentCommit/Push do NOT participate in a -// real version-control commit graph (currently SQLite). cmd/bd's PostRun +// real version-control commit graph (e.g. Postgres, SQLite). cmd/bd's PostRun // skips its Dolt-only maintenance tail (auto-commit, auto-export, auto-backup, // auto-push) when the active store implements this and returns true. // // The marker is negative and lives only on non-Dolt backends by design: a store // that does NOT implement it is assumed to be Dolt, so the default path runs the -// full maintenance tail unchanged and stays byte-identical. A future non-Dolt -// implementation would opt out of Dolt maintenance by implementing this marker. +// full maintenance tail unchanged and stays byte-identical. New SQL backends opt +// OUT of Dolt maintenance by implementing this; they never have to touch the +// Dolt constructors. type NonCommitGraphBackend interface { CommitGraphUnsupported() bool } diff --git a/internal/storage/conformance/audit_config_metadata_slots_repomtime.go b/internal/storage/conformance/audit_config_metadata_slots_repomtime.go index 116891e02..5dc11eba2 100644 --- a/internal/storage/conformance/audit_config_metadata_slots_repomtime.go +++ b/internal/storage/conformance/audit_config_metadata_slots_repomtime.go @@ -43,8 +43,9 @@ func auditFoldCount(all map[string]string, want string) int { // testAuditConfigKeyCaseSensitive pins the reference's config key-column collation. // The finding predicted Dolt's VARCHAR primary key would be case-insensitive, but // the embedded-Dolt oracle treats "MyKey" and "mykey" as DISTINCT keys: each holds -// its own value and GetAllConfig carries both rows. SQLite's BINARY collation matches -// that behavior. +// its own value and GetAllConfig carries both rows. Postgres text and SQLite BINARY +// are case-sensitive too, so they match; the divergence suspect is real MySQL, whose +// server-default utf8mb4 collation is case-insensitive and would collapse the two. func testAuditConfigKeyCaseSensitive(t *testing.T, f Factory) { s := f(t) c := ctx() diff --git a/internal/storage/conformance/audit_labels-comments-events.go b/internal/storage/conformance/audit_labels-comments-events.go index c842c8fab..8f2088cba 100644 --- a/internal/storage/conformance/audit_labels-comments-events.go +++ b/internal/storage/conformance/audit_labels-comments-events.go @@ -245,7 +245,7 @@ func testAuditImportCommentSubSecond(t *testing.T, f Factory) { c := ctx() must(t, s.CreateIssue(c, withDefaults(&types.Issue{ID: "test-i", Title: "I"}), "a")) // Sub-second precision is intentionally backend-specific (Dolt datetime(0) rounds - // half-up; SQLite keeps the fraction), so we test the + // half-up; SQLite keeps the fraction; Postgres keeps microseconds), so we test the // portable contract every backend honors: a whole-second UTC timestamp round-trips // exactly. Higher fidelity on some backends is accepted, not asserted. ts := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) diff --git a/internal/storage/conformance/audit_molecule_wisp_batch_iter.go b/internal/storage/conformance/audit_molecule_wisp_batch_iter.go index c56457bde..05c0ca51f 100644 --- a/internal/storage/conformance/audit_molecule_wisp_batch_iter.go +++ b/internal/storage/conformance/audit_molecule_wisp_batch_iter.go @@ -665,8 +665,9 @@ func testAuditGetNextChildID(t *testing.T, f Factory) { } } -// The self-heal scan (id LIKE parent.'%') is case-sensitive on Dolt and SQLite -// (see bd-oyvc2.10), so a different-cased sibling parent's children never advance +// The self-heal scan (id LIKE parent.'%') is case-sensitive on every backend +// (Dolt/MySQL bin collation, Postgres "C", SQLite case_sensitive_like β€” see +// bd-oyvc2.10), so a different-cased sibling parent's children never advance // this parent's counter. func testAuditGetNextChildIDCaseSensitive(t *testing.T, f Factory) { s := f(t) diff --git a/internal/storage/conformance/audit_search-counts-stats.go b/internal/storage/conformance/audit_search-counts-stats.go index 756769a1d..13acbde54 100644 --- a/internal/storage/conformance/audit_search-counts-stats.go +++ b/internal/storage/conformance/audit_search-counts-stats.go @@ -182,7 +182,7 @@ func testAuditReadyWorkDepCreatedAtParity(t *testing.T, f Factory) { } // countByColumnInTx emits COALESCE(priority, ”) GROUP BY priority; priority is -// integer NOT NULL. Both maintained implementations return a string key, then +// integer NOT NULL. MySQL/Dolt coerce the int to a string key, then // countGroupForTablesInTx prepends 'P'. Reference keys: P0/P1/P2. func testAuditCountByPriority(t *testing.T, f Factory) { s := f(t) @@ -366,7 +366,7 @@ func testAuditSearchIdenticalTimestampIDOrder(t *testing.T, f Factory) { } // SortBy="closed" emits ORDER BY closed_at DESC, id ASC. closed_at is nullable; -// the storage contract places NULLs last on DESC. The open (NULL-closed) rows follow the +// MySQL/Dolt place NULLs LAST on DESC. The open (NULL-closed) rows follow the // closed ones, ordered by id ASC. func testAuditSearchSortByClosedNullsLast(t *testing.T, f Factory) { s := f(t) @@ -409,9 +409,9 @@ func testAuditSearchTextIDBranchExternalRef(t *testing.T, f Factory) { } } -// LIKE over raw-cased operands is case-sensitive on both implementations: Dolt uses -// a binary table collation, and SQLite sets PRAGMA case_sensitive_like (sqlite/dsn.go). -// SQLite's default LIKE is +// LIKE over raw-cased operands is case-sensitive on every backend: Dolt/MySQL use +// the utf8mb4_0900_bin table collation, Postgres uses collation "C", and SQLite +// sets PRAGMA case_sensitive_like (sqlite/dsn.go) β€” its default LIKE is // ASCII-case-insensitive and silently diverged (bd-oyvc2.10). IDPrefix filtering // (sqlbuild/filter.go `id LIKE ?`) must therefore distinguish test-AB from test-ab. func testAuditSearchIDPrefixCaseSensitive(t *testing.T, f Factory) { diff --git a/internal/storage/conformance/claim.go b/internal/storage/conformance/claim.go index 5684e5c28..0dbc9ca05 100644 --- a/internal/storage/conformance/claim.go +++ b/internal/storage/conformance/claim.go @@ -2,6 +2,8 @@ package conformance import ( "errors" + "fmt" + "sync" "testing" "time" @@ -10,10 +12,10 @@ import ( "github.com/steveyegge/beads/internal/types" ) -// Claim / lease behavior (Gas Station v1.1 dead-worker recovery). Each backend +// Claim / lease behavior (Gas Station v1.1 dead-worker recovery). Every backend // routes ClaimIssue/ClaimReadyIssue/HeartbeatIssue/ReclaimExpiredLeases through the -// shared issueops implementations, so these assertions hold identically on Dolt and -// SQLite. issueops.WithLeaseTTL pins the lease deterministically +// shared issueops implementations, so these assertions hold identically on the Dolt +// reference and every SQL backend. issueops.WithLeaseTTL pins the lease deterministically // so the reclaim path is testable without wall-clock waits. // testClaim: claiming an open issue stamps assignee, in_progress, started_at, and a @@ -154,6 +156,102 @@ func issueID(i *types.Issue) string { return i.ID } +// requireMultiWriter skips the caller unless the backend backs concurrent writers on +// separate connections. Postgres and MySQL use a multi-connection pool; SQLite pins its +// pool to a single connection (sqliteDialect.Open sets MaxOpenConns(1)) and embedded-Dolt +// serializes writers, so a "concurrent" claim race there only ever exercises the trivial +// serialized path. The dialect name is the profile signal: sqlkit-backed SQL stores +// expose DialectName(); the Dolt reference does not, so it skips via the type assertion. +func requireMultiWriter(t *testing.T, s storage.DoltStorage) { + t.Helper() + dn, ok := s.(interface{ DialectName() string }) + if !ok { + t.Skip("backend does not expose a SQL dialect; concurrent multi-writer claim race not applicable") + } + switch dn.DialectName() { + case "postgres", "mysql": + // multi-connection pool: a genuine cross-connection race. + default: + t.Skipf("dialect %q is single-writer (pool pinned to one connection); concurrent claim race not applicable", dn.DialectName()) + } +} + +// testClaimReadyIssueConcurrentExclusivity races many workers, each on its own pooled +// connection, to claim a small pool of ready issues and asserts every issue is claimed by +// exactly one worker. This is the multi-writer property the SQL backends exist for β€” the +// whole reason to run bd on Postgres/MySQL instead of single-writer Dolt β€” and the shared +// claim path's cross-process exclusivity (a conditional-UPDATE CAS: only the writer whose +// UPDATE finds the row still unassigned wins; the losers' ClaimReadyIssue retries the next +// ready candidate) is otherwise only asserted, never raced. Gated to the multi-connection +// backends; see requireMultiWriter. +func testClaimReadyIssueConcurrentExclusivity(t *testing.T, f Factory) { + s := f(t) + requireMultiWriter(t, s) + + const readyCount = 8 + for i := 0; i < readyCount; i++ { + must(t, s.CreateIssue(ctx(), withDefaults(&types.Issue{ + ID: fmt.Sprintf("ccx-%d", i), + Title: "ready", + Priority: 1, + }), "a")) + } + + const workers = 24 // > readyCount, so workers must contend and most must lose a race + + var ( + mu sync.Mutex + claims = make(map[string]string) // issueID -> claiming worker + dupes []string + errs []error + ) + start := make(chan struct{}) + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func(worker string) { + defer wg.Done() + <-start // barrier: release all workers together to maximize contention + // Drain: keep claiming until no ready work remains. Bounded so a bug that + // kept returning issues can never hang the suite. + for attempt := 0; attempt < readyCount+workers; attempt++ { + claimed, err := s.ClaimReadyIssue(ctx(), types.WorkFilter{}, worker) + if err != nil { + mu.Lock() + errs = append(errs, err) + mu.Unlock() + return + } + if claimed == nil { + return // no ready work left to claim + } + if claimed.Assignee != worker || claimed.Status != types.StatusInProgress { + t.Errorf("claimed %s not owned by claimer: assignee=%q status=%q", claimed.ID, claimed.Assignee, claimed.Status) + } + mu.Lock() + if prev, ok := claims[claimed.ID]; ok { + dupes = append(dupes, fmt.Sprintf("%s claimed by both %s and %s", claimed.ID, prev, worker)) + } else { + claims[claimed.ID] = worker + } + mu.Unlock() + } + }(fmt.Sprintf("worker-%d", w)) + } + close(start) + wg.Wait() + + for _, e := range errs { + t.Errorf("ClaimReadyIssue errored under concurrency: %v", e) + } + for _, d := range dupes { + t.Errorf("claim exclusivity violated β€” %s", d) + } + if len(claims) != readyCount { + t.Errorf("distinct claimed issues = %d, want %d (every ready issue claimed exactly once, none lost)", len(claims), readyCount) + } +} + // testHeartbeatRenewsLease: a heartbeat extends the lease (and keeps the claim). // Heartbeating with a one-hour TTL must push lease_expires_at far into the future. func testHeartbeatRenewsLease(t *testing.T, f Factory) { diff --git a/internal/storage/conformance/conformance.go b/internal/storage/conformance/conformance.go index 9c625e2f3..242e3ea87 100644 --- a/internal/storage/conformance/conformance.go +++ b/internal/storage/conformance/conformance.go @@ -149,6 +149,7 @@ func RunAll(t *testing.T, factory Factory) { t.Run("ClaimNotClaimable", func(t *testing.T) { testClaimNotClaimable(t, factory) }) t.Run("ClaimReadyIssue", func(t *testing.T) { testClaimReadyIssue(t, factory) }) t.Run("ClaimReadyIssueLabelFilters", func(t *testing.T) { testClaimReadyIssueLabelFilters(t, factory) }) + t.Run("ClaimReadyIssueConcurrentExclusivity", func(t *testing.T) { testClaimReadyIssueConcurrentExclusivity(t, factory) }) t.Run("HeartbeatRenewsLease", func(t *testing.T) { testHeartbeatRenewsLease(t, factory) }) t.Run("HeartbeatWisp", func(t *testing.T) { testHeartbeatWisp(t, factory) }) t.Run("ReclaimExpiredLease", func(t *testing.T) { testReclaimExpiredLease(t, factory) }) @@ -190,10 +191,12 @@ func RunAll(t *testing.T, factory Factory) { t.Run("Transaction", func(t *testing.T) { testTransaction(t, factory) }) } -// RunDeferredReads runs the subset of the suite covering SQLite's shared -// non-version-control reads: statistics, external-ref lookup, and staleness. RunAll -// remains the full Dolt reference; SQLite runs this focused gate for methods supplied -// by issueops while its Dolt-only methods fail loudly as unsupported. +// RunDeferredReads runs the subset of the suite covering the shared "deferred" +// non-version-control reads β€” statistics, external-ref lookup, and staleness β€” that +// the SQL-family backends (postgres/mysql/sqlite) implement through issueops. RunAll +// is the full fail-loud measurement (and stays red on genuinely Dolt-only methods +// like slots), so these backends run this focused GREEN gate instead. The Dolt +// reference covers the same cases via RunAll. func RunDeferredReads(t *testing.T, factory Factory) { t.Helper() t.Run("Statistics", func(t *testing.T) { testStatistics(t, factory) }) diff --git a/internal/storage/conformance/portable.go b/internal/storage/conformance/portable.go index 990520a51..7cd0afaba 100644 --- a/internal/storage/conformance/portable.go +++ b/internal/storage/conformance/portable.go @@ -12,9 +12,9 @@ import ( "github.com/steveyegge/beads/internal/types" ) -// This file holds the behavior contract for the portable, non-version-control methods -// that SQLite implements through shared issueops helpers: molecule rollups, repo-mtime -// cache, event/dependency streams, dependent +// This file holds the behavior contract for the "portable" non-version-control methods +// that the SQL-family backends (postgres/mysql/sqlite) implement through shared issueops +// helpers β€” molecule rollups, repo-mtime cache, event/dependency streams, dependent // counts, wisp cascade discovery, comment/audit writes, id rekey, source-repo purge, and // batch create. Each case is validated against the embedded-Dolt reference (the oracle) // and, once a method is wired into a backend, must match it there too. @@ -23,7 +23,7 @@ import ( // ties, tie-broken "current step") is asserted as a set, never positionally. // RunPortableMethods runs the portable-method behavior contract. The Dolt reference runs -// it via RunAll; SQLite runs it for the methods supplied by the shared layer. +// it via RunAll; each SQL backend runs it once the methods are implemented. func RunPortableMethods(t *testing.T, factory Factory) { t.Helper() t.Run("MoleculeProgress", func(t *testing.T) { testGetMoleculeProgress(t, factory) }) diff --git a/internal/storage/dolt/open.go b/internal/storage/dolt/open.go index 32f8fe122..7cc664cb7 100644 --- a/internal/storage/dolt/open.go +++ b/internal/storage/dolt/open.go @@ -48,25 +48,6 @@ func ApplyCLIAutoStart(beadsDir string, cfg *Config) { cfg.AutoStart = resolveAutoStart(true, autoStartCfg, mode) } -// requireDoltBackend keeps metadata-driven callers from bypassing the storage -// factory and interpreting another backend's workspace as Dolt. Removed backend -// identifiers deliberately remain recognizable in metadata so this check can fail -// closed instead of opening a new, empty Dolt database. -func requireDoltBackend(fileCfg *configfile.Config) error { - switch fileCfg.Backend { - case configfile.BackendPostgres, configfile.BackendMySQL: - return fmt.Errorf("configured storage backend %q is no longer supported and cannot be opened as Dolt: %s", fileCfg.Backend, configfile.RemovedBackendDetail(fileCfg.Backend)) - } - if !configfile.IsSupportedBackend(fileCfg.Backend) { - return fmt.Errorf("configured storage backend %q in metadata.json is not recognized and cannot be opened as Dolt; %s", fileCfg.Backend, configfile.BackendNotOpenedGuarantee) - } - backend := fileCfg.GetBackend() - if backend != configfile.BackendDolt { - return fmt.Errorf("configured storage backend %q cannot be opened as Dolt", backend) - } - return nil -} - // NewFromConfig creates a DoltStore based on the metadata.json configuration. // beadsDir is the path to the .beads directory. func NewFromConfig(ctx context.Context, beadsDir string) (*DoltStore, error) { @@ -85,9 +66,6 @@ func NewFromConfigWithCLIOptions(ctx context.Context, beadsDir string, cfg *Conf if fileCfg == nil { fileCfg = configfile.DefaultConfig() } - if err := requireDoltBackend(fileCfg); err != nil { - return nil, err - } // Apply central server config as defaults for any server fields not // set in the per-project metadata.json. This eliminates the need to @@ -115,9 +93,6 @@ func NewFromConfigWithOptions(ctx context.Context, beadsDir string, cfg *Config) if fileCfg == nil { fileCfg = configfile.DefaultConfig() } - if err := requireDoltBackend(fileCfg); err != nil { - return nil, err - } // Apply central server config as defaults for any server fields not // set in the per-project metadata.json. diff --git a/internal/storage/dolt/open_test.go b/internal/storage/dolt/open_test.go index 9a3228d8b..2e5b0b9af 100644 --- a/internal/storage/dolt/open_test.go +++ b/internal/storage/dolt/open_test.go @@ -162,50 +162,6 @@ func TestApplyCLIAutoStart_DisableAutoStartWins(t *testing.T) { } } -func TestConfigConstructorsRejectNonDoltBackends(t *testing.T) { - for _, backend := range []string{ - configfile.BackendSQLite, - configfile.BackendPostgres, - configfile.BackendMySQL, - "mystery", - } { - t.Run(backend, func(t *testing.T) { - beadsDir := t.TempDir() - if err := (&configfile.Config{Backend: backend}).Save(beadsDir); err != nil { - t.Fatalf("save metadata: %v", err) - } - - constructors := map[string]func() (*DoltStore, error){ - "standard": func() (*DoltStore, error) { - return NewFromConfigWithOptions(t.Context(), beadsDir, &Config{DisableAutoStart: true}) - }, - "cli": func() (*DoltStore, error) { - return NewFromConfigWithCLIOptions(t.Context(), beadsDir, &Config{DisableAutoStart: true}) - }, - } - for name, construct := range constructors { - t.Run(name, func(t *testing.T) { - store, err := construct() - if store != nil { - _ = store.Close() - t.Fatalf("non-Dolt backend %q unexpectedly opened as Dolt", backend) - } - if err == nil || !strings.Contains(err.Error(), "cannot be opened as Dolt") { - t.Fatalf("constructor error = %v, want backend mismatch", err) - } - if backend == configfile.BackendPostgres || backend == configfile.BackendMySQL { - for _, want := range []string{"no longer supported", "was not opened or modified", "bd help init-safety"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("removed-backend constructor error missing %q: %v", want, err) - } - } - } - }) - } - }) - } -} - func TestCLIDirUsesSharedDoltRootInSharedServerMode(t *testing.T) { sharedRoot := t.TempDir() t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") diff --git a/internal/storage/embeddeddolt/iter_stubs.go b/internal/storage/embeddeddolt/iter_stubs.go index 39f8e9a4b..cb5f76fcb 100644 --- a/internal/storage/embeddeddolt/iter_stubs.go +++ b/internal/storage/embeddeddolt/iter_stubs.go @@ -4,8 +4,8 @@ // // Slice-wrapping stubs for the Iter* methods. The embedded Dolt backend // uses a per-method short-lived connection model (`withConn`) which is -// incompatible with the dedicated-connection cursor pattern used by the -// streaming iterators in internal/storage/dolt. +// incompatible with the dedicated-conn cursor pattern used by the +// streaming iterators in internal/storage/dolt and internal/storage/postgres. // The interface ships complete now (be-jaavsb / be-yinl4d); a follow-up // child of be-yinl4d may add a streaming variant if EmbeddedDoltStore // gains a cursor-conn API. For now every Iter* method materializes the diff --git a/internal/storage/errors_unsupported.go b/internal/storage/errors_unsupported.go index 6867b739a..fd4c70359 100644 --- a/internal/storage/errors_unsupported.go +++ b/internal/storage/errors_unsupported.go @@ -7,7 +7,7 @@ import "fmt" // taxonomy; this is the embryo of the seam's ErrUnsupported{Op, Backend}. type ErrUnsupported struct { Op string // method name, e.g. "AddLabel" or "Transaction.CreateIssues" - Backend string // e.g. "sqlite" + Backend string // e.g. "postgres" } func (e *ErrUnsupported) Error() string { diff --git a/internal/storage/issueops/blocked_state.go b/internal/storage/issueops/blocked_state.go index 79a37065a..61090dd9f 100644 --- a/internal/storage/issueops/blocked_state.go +++ b/internal/storage/issueops/blocked_state.go @@ -334,6 +334,29 @@ func unmarkBlockedTemplateForWisps() string { `, waitsForGateBlockedSQL) } +// SelfReferentialRecomputeTemplates returns the four is_blocked recompute UPDATE +// statements exactly as issueops emits them to the driver (with a representative +// IN-clause batch substituted for the placeholder). Each one updates issues/wisps +// while its WHERE clause self-references that same target table in EXISTS subqueries, +// which MySQL 8 rejects with error 1093 unless the mysqldialect driver rewrites the +// statement into a materialized derived-table form. +// +// It is exported solely so that rewrite can be regression-tested against these real +// templates instead of a hand-copied fixture: if a template ever drifts out of the +// shape the rewrite recognizes (a predicate before `id IN`, a backticked table, a +// renamed alias, changed whitespace), the mysqldialect rewrite test fails in a normal +// non-gated `go test` run rather than only under the env-gated MySQL conformance suite. +func SelfReferentialRecomputeTemplates() []string { + placeholders, _ := buildSQLInClause([]string{"a", "b", "c"}) + render := func(tmpl string) string { return fmt.Sprintf(tmpl, placeholders) } + return []string{ + render(markBlockedTemplateForIssues()), + render(unmarkBlockedTemplateForIssues()), + render(markBlockedTemplateForWisps()), + render(unmarkBlockedTemplateForWisps()), + } +} + //nolint:gosec // G201: callers pass constant templates; only IN-clause placeholders are formatted in. func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl, unmarkTmpl string, ids []string) (int64, error) { var changed int64 diff --git a/internal/storage/issueops/count.go b/internal/storage/issueops/count.go index 71d7fcd3a..92c4d9d21 100644 --- a/internal/storage/issueops/count.go +++ b/internal/storage/issueops/count.go @@ -186,7 +186,9 @@ func countByColumnInTx(ctx context.Context, tx *sql.Tx, filter types.IssueFilter whereSQL = " WHERE " + strings.Join(clauses, " AND ") } // CAST the group column to text before COALESCE so an integer column (priority) - // COALESCEs with '' consistently across Dolt and SQLite. + // COALESCEs with '' consistently. MySQL/Dolt coerce int->text implicitly, but + // Postgres rejects COALESCE(integer, '') (22P02); CAST(... AS CHAR) is portable + // (pgdialect -> ::text, native on MySQL/SQLite) and byte-identical on every backend. //nolint:gosec // G201: tables.Main hardcoded; col validated by caller query := fmt.Sprintf("SELECT COALESCE(CAST(%s AS CHAR), ''), COUNT(*) FROM %s%s GROUP BY %s", col, tables.Main, whereSQL, col) rows, err := tx.QueryContext(ctx, query, args...) diff --git a/internal/storage/issueops/create.go b/internal/storage/issueops/create.go index 5f3fdb9ae..4ee6fe247 100644 --- a/internal/storage/issueops/create.go +++ b/internal/storage/issueops/create.go @@ -895,9 +895,10 @@ func ReconcileChildCounters(ctx context.Context, tx *sql.Tx, issues []*types.Iss if err == nil && current >= b.maxChild { continue } - // Qualify the existing-row column with the table name so the canonical - // MySQL form and SQLite's translated ON CONFLICT form both unambiguously - // refer to the target row rather than the incoming value. + // Qualify the existing-row column with the table name. Bare `last_child` + // on the update RHS is a valid MySQL "current row value" reference, but + // Postgres sees it as ambiguous between the target row and EXCLUDED + // (SQLSTATE 42702); table-qualifying it is unambiguous on every backend. //nolint:gosec // G201: table is one of two hardcoded constants. if _, err := tx.ExecContext(ctx, fmt.Sprintf(` INSERT INTO %[1]s (parent_id, last_child) VALUES (?, ?) diff --git a/internal/storage/issueops/helpers.go b/internal/storage/issueops/helpers.go index b10da15c1..b06747209 100644 --- a/internal/storage/issueops/helpers.go +++ b/internal/storage/issueops/helpers.go @@ -90,9 +90,10 @@ func issueUpsertAssignments(table string, rejectStaleUpdate bool) string { assignments := make([]string, 0, len(issueUpsertColumns)) for _, col := range issueUpsertColumns { if rejectStaleUpdate { - // Qualify existing-row references with the table name so the target value - // remains unambiguous after the SQLite upsert translation. VALUES(...) is - // the incoming row in the canonical Dolt/MySQL-dialect statement. + // Qualify the existing-row references with the table name. Postgres's + // ON CONFLICT DO UPDATE rejects a bare `updated_at` as ambiguous (it could + // be the target row or EXCLUDED); .updated_at is unambiguous and is + // also accepted by MySQL/Dolt/SQLite. VALUES(...) is the incoming row. assignments = append(assignments, fmt.Sprintf("%s = IF(VALUES(updated_at) > %s.updated_at, VALUES(%s), %s.%s)", col, table, col, table, col)) } else { diff --git a/internal/storage/issueops/lease.go b/internal/storage/issueops/lease.go index 653edf0b6..031ccfd64 100644 --- a/internal/storage/issueops/lease.go +++ b/internal/storage/issueops/lease.go @@ -116,8 +116,9 @@ func HeartbeatIssueInTx(ctx context.Context, tx DBTX, id, actor string) error { now := time.Now().UTC() leaseClause, leaseArgs := leaseSetClause(now, leaseTTL(ctx)) - // Stamp updated_at = now on the heartbeat. Dolt's ON UPDATE CURRENT_TIMESTAMP - // trigger does this automatically; SQLite has no such trigger, so without an explicit stamp an + // Stamp updated_at = now on the heartbeat. On Dolt/MySQL the issues/wisps + // ON UPDATE CURRENT_TIMESTAMP trigger bumps updated_at on every heartbeat; + // Postgres and SQLite have no such trigger, so without an explicit stamp an // actively-heartbeated issue keeps a stale updated_at and bd stale (which // filters in_progress rows on updated_at < cutoff) diverges from the Dolt // oracle. Claim already stamps updated_at explicitly, so this is heartbeat-only. diff --git a/internal/storage/iter.go b/internal/storage/iter.go index 848a593a7..365ff27ab 100644 --- a/internal/storage/iter.go +++ b/internal/storage/iter.go @@ -2,7 +2,7 @@ // // Iter[T] is a generic streaming-iterator shape that lets storage callers // walk an unbounded result set without materializing the whole slice. -// Per-method implementations live in the concrete backend packages. +// Per-method implementations live in the dolt/ and postgres/ sub-packages. // // See be-jaavsb (design) and be-yinl4d (architecture) for the motivation. package storage diff --git a/internal/storage/mysql/completeness_test.go b/internal/storage/mysql/completeness_test.go new file mode 100644 index 000000000..7fd092bed --- /dev/null +++ b/internal/storage/mysql/completeness_test.go @@ -0,0 +1,107 @@ +package mysql + +import ( + "os" + "regexp" + "sort" + "testing" + + "github.com/steveyegge/beads/internal/storage/conformance" +) + +// legitimatelyUnsupported is the EXPLICIT denominator of every storage.DoltStorage +// method the Postgres wedge deliberately does not implement, each with a reason. +// +// The interface-completeness gate (TestInterfaceCompleteness) asserts the +// generated capability shell (unsupported_gen.go) contains EXACTLY these methods +// β€” no more, no less. This is the guard the red-team had to stand in for: a +// method that silently resolves to the typed-unsupported shell (the class that +// bit us: DeleteIssues, GetCustomStatuses, the batch reads, …) fails this test +// at build time instead of failing a user at runtime. Adding a method to the +// shell without an entry here is a hard error that forces the triage question: +// "implement it in sqlkit, or record here why the wedge legitimately omits it." +var legitimatelyUnsupported = map[string]string{ + // VersionControl β€” the Dolt commit graph. The wedge's whole premise is + // "bd on Postgres, history/version-control simply absent". + "Branch": "VC: Dolt branch", "Checkout": "VC: Dolt checkout", + "CurrentBranch": "VC", "DeleteBranch": "VC", "ListBranches": "VC", + // Commit/CommitWithConfig/CommitMergeResolution are NO-OPS on Postgres (data + // is already durable per-tx), not unsupported β€” implemented on postgres.Store. + "CommitExists": "VC", "GetCurrentCommit": "VC: Dolt hash", "Status": "VC", + "Log": "VC: dolt_log", "Merge": "VC", "GetConflicts": "VC", "ResolveConflicts": "VC", + + // HistoryViewer β€” Dolt time-travel. + "History": "history: dolt_log", "AsOf": "history: as-of ref", "Diff": "history: dolt_diff", + + // RemoteStore β€” Dolt remotes. + "AddRemote": "remote", "RemoveRemote": "remote", "HasRemote": "remote", "ListRemotes": "remote", + "Push": "remote", "Pull": "remote", "ForcePush": "remote", "PushRemote": "remote", + "PullRemote": "remote", "Fetch": "remote", "PushTo": "remote", "PullFrom": "remote", + + // SyncStore β€” Dolt peer sync. + "Sync": "sync", "SyncStatus": "sync", + + // FederationStore β€” Dolt-remote-backed peer table. + "AddFederationPeer": "federation", "GetFederationPeer": "federation", + "ListFederationPeers": "federation", "RemoveFederationPeer": "federation", + + // CompactionStore β€” bound to Dolt commit hashes. + "CheckEligibility": "compaction", "ApplyCompaction": "compaction", + "GetTier1Candidates": "compaction", "GetTier2Candidates": "compaction", + "SnapshotIssue": "compaction", "GetCompactionSnapshot": "compaction", "RestoreFromSnapshot": "compaction", + + // MergeSlot β€” the Dolt merge-serialization primitive. + "MergeSlotCreate": "merge-slot", "MergeSlotCheck": "merge-slot", + "MergeSlotAcquire": "merge-slot", "MergeSlotRelease": "merge-slot", +} + +var shellMethodRe = regexp.MustCompile(`func \(unsupportedDoltStorage\) ([A-Za-z0-9]+)\(`) + +// TestInterfaceCompleteness is the interface-completeness gate: the generated +// shell must equal the audited legitimatelyUnsupported set, both directions. +func TestInterfaceCompleteness(t *testing.T) { + src, err := os.ReadFile("unsupported_gen.go") + if err != nil { + t.Fatalf("read unsupported_gen.go: %v", err) + } + shell := map[string]bool{} + for _, m := range shellMethodRe.FindAllStringSubmatch(string(src), -1) { + shell[m[1]] = true + } + if len(shell) == 0 { + t.Fatal("parsed 0 shell methods β€” regex or file drift") + } + + // (1) No SILENT gap: every shell method must be explicitly justified. + var unjustified []string + for m := range shell { + if _, ok := legitimatelyUnsupported[m]; !ok { + unjustified = append(unjustified, m) + } + } + sort.Strings(unjustified) + for _, m := range unjustified { + t.Errorf("method %q resolves to the typed-unsupported shell but is NOT in legitimatelyUnsupported: implement it on *sqlkit.Store, or add it here with a reason", m) + } + + // (2) No STALE allowlist: every justified method must still be in the shell + // (else it was implemented and the entry should be removed). + var stale []string + for m := range legitimatelyUnsupported { + if !shell[m] { + stale = append(stale, m) + } + } + sort.Strings(stale) + for _, m := range stale { + t.Errorf("method %q is in legitimatelyUnsupported but no longer in the shell (implemented?): remove the allowlist entry", m) + } +} + +// TestUnsupportedContract is the behavioral complement to TestInterfaceCompleteness: +// every allowlisted method must actually return a typed storage.ErrUnsupported when +// called, not panic or return a different error. DB-free β€” the generated stubs ignore +// their receiver, so a zero-value store answers them. +func TestUnsupportedContract(t *testing.T) { + conformance.RunUnsupportedContract(t, &Store{}, legitimatelyUnsupported) +} diff --git a/internal/storage/mysql/conformance_test.go b/internal/storage/mysql/conformance_test.go new file mode 100644 index 000000000..e7946f517 --- /dev/null +++ b/internal/storage/mysql/conformance_test.go @@ -0,0 +1,80 @@ +package mysql + +import ( + "context" + "database/sql" + "fmt" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/conformance" +) + +// TestConformance runs bd's backend-agnostic storage conformance suite against the +// MySQL backend. Every failure is a MySQL gap: an allowlisted-unsupported method +// (expected β€” completeness_test.go) or a latent divergence. Gated on +// BEADS_MYSQL_TEST_URL (a server DSN, e.g. user:pass@tcp(host:3306)/). +func TestConformance(t *testing.T) { + url := os.Getenv("BEADS_MYSQL_TEST_URL") + if url == "" { + t.Skip("BEADS_MYSQL_TEST_URL not set") + } + conformance.RunAll(t, mysqlConformanceFactory(url)) +} + +// TestDeferredReads runs the shared deferred non-version-control reads (statistics, +// external-ref, stale) that the backend now implements through issueops, as a GREEN +// gate. RunAll stays the fail-loud measurement (red on genuinely Dolt-only methods +// like slots), so this focused gate is what conformance.sh runs. Gated on +// BEADS_MYSQL_TEST_URL. +func TestDeferredReads(t *testing.T) { + url := os.Getenv("BEADS_MYSQL_TEST_URL") + if url == "" { + t.Skip("BEADS_MYSQL_TEST_URL not set") + } + conformance.RunDeferredReads(t, mysqlConformanceFactory(url)) +} + +// TestPortableMethods runs the full portable-method behavior contract. Red until the +// methods are wired into sqlkit; green after. Gated on BEADS_MYSQL_TEST_URL. +func TestPortableMethods(t *testing.T) { + url := os.Getenv("BEADS_MYSQL_TEST_URL") + if url == "" { + t.Skip("BEADS_MYSQL_TEST_URL not set") + } + conformance.RunPortableMethods(t, mysqlConformanceFactory(url)) +} + +// mysqlConformanceFactory provisions a fresh, isolated database per sub-test against +// the MySQL server at url, seeded with issue_prefix as `bd init` leaves it. +func mysqlConformanceFactory(url string) conformance.Factory { + base := time.Now().UnixNano() + var seq int64 + return func(t *testing.T) storage.DoltStorage { + ctx := context.Background() + database := fmt.Sprintf("conf_%d_%d", base, atomic.AddInt64(&seq, 1)) + + st, err := Provision(ctx, url, database) + if err != nil { + t.Fatalf("Provision: %v", err) + } + // The conformance factory contract wants an init'd store (issue_prefix set), + // as `bd init` leaves it. No Commit β€” MySQL has no commit graph. + if err := st.SetConfig(ctx, "issue_prefix", "test"); err != nil { + t.Fatalf("SetConfig(issue_prefix): %v", err) + } + t.Cleanup(func() { + if serverDSN, e := withDatabase(url, ""); e == nil { + if srv, e2 := sql.Open("mysql", serverDSN); e2 == nil { + _, _ = srv.ExecContext(context.Background(), "DROP DATABASE IF EXISTS `"+database+"`") + _ = srv.Close() + } + } + _ = st.Close() + }) + return st + } +} diff --git a/internal/storage/mysql/credential.go b/internal/storage/mysql/credential.go new file mode 100644 index 000000000..7026176c3 --- /dev/null +++ b/internal/storage/mysql/credential.go @@ -0,0 +1,67 @@ +package mysql + +import ( + "context" + "fmt" + "os" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/creds" + "github.com/steveyegge/beads/internal/storage/mysqldialect" +) + +// resolveDSNCredential returns baseDSN with a credential resolved and placed, ready to +// connect. The precedence, high to low: +// +// - BEADS_MYSQL_CREDENTIAL_COMMAND set is a hard error: an identity credential is +// presented as a username to a gateway, and this backend connects directly; +// - the DSN already carries a password (a full BEADS_MYSQL_URL, or an explicit +// --mysql-url) β€” it wins outright, no ladder is applied; +// - BEADS_MYSQL_PASSWORD_COMMAND β€” a credential command (e.g. Vault / RDS-IAM / GCP-IAM); +// run at open time, cached until near expiry; +// - BEADS_MYSQL_PASSWORD β€” a static password; +// - the credentials file [host:port] ($BEADS_CREDENTIALS_FILE or +// ~/.config/beads/credentials); +// - nothing configured β€” baseDSN is returned untouched and the connection is +// attempted with an empty password (valid only for an unsecured dev server; +// go-sql-driver reads no env vars or option files, so there is no ~/.my.cnf +// fallback). +// +// It fails closed: if a configured credential command errors, the open aborts β€” it +// never silently downgrades to a lower rung. +func resolveDSNCredential(ctx context.Context, cfg *configfile.Config, baseDSN string) (string, error) { + if os.Getenv("BEADS_MYSQL_CREDENTIAL_COMMAND") != "" { + return "", fmt.Errorf("mysql: BEADS_MYSQL_CREDENTIAL_COMMAND is reserved: an identity credential is presented as a username to a gateway, and the mysql backend connects directly to the database with no gateway to present it to; use BEADS_MYSQL_PASSWORD_COMMAND for a helper that produces a password") + } + // host:port keys the credentials-file lookup; ok=false (unix socket / unparseable) + // leaves the file rung not-configured (zero Host/Port), never failing. + host, port, _ := mysqldialect.HostPort(baseDSN) + return resolveDSNWithSources(ctx, baseDSN, + creds.CommandSource{Command: cfg.GetMySQLPasswordCommand(), Kind: creds.KindSecret, Label: "BEADS_MYSQL_PASSWORD_COMMAND"}, + creds.EnvSource{Var: "BEADS_MYSQL_PASSWORD"}, + creds.FileSource{Host: host, Port: port, Lookup: configfile.LookupCredentialsPassword, Label: "credentials-file"}, + ) +} + +// resolveDSNWithSources is the testable core of resolveDSNCredential: it applies the +// given credential ladder to baseDSN. +func resolveDSNWithSources(ctx context.Context, baseDSN string, sources ...creds.Source) (string, error) { + if mysqldialect.HasPassword(baseDSN) { + return baseDSN, nil + } + cred, ok, err := creds.ResolveLadder(ctx, sources...) + if err != nil { + return "", err + } + if !ok { + return baseDSN, nil + } + // The SQL backends take a secret in the password slot; an identity credential + // (presented as a username to a gateway) has no home on a direct connection, so + // refuse it loudly rather than land a token where a password belongs. Defense in + // depth behind the BEADS_MYSQL_CREDENTIAL_COMMAND reject above. + if cred.Kind != creds.KindSecret { + return "", fmt.Errorf("mysql: credential from %s is an identity, not a password; the mysql backend connects directly and has no gateway to present it to", cred.Source) + } + return mysqldialect.WithCredential(baseDSN, cred.Username, cred.Value) +} diff --git a/internal/storage/mysql/credential_test.go b/internal/storage/mysql/credential_test.go new file mode 100644 index 000000000..5d488a0ba --- /dev/null +++ b/internal/storage/mysql/credential_test.go @@ -0,0 +1,288 @@ +package mysql + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + gomysql "github.com/go-sql-driver/mysql" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/creds" +) + +const baseDSN = "bts@tcp(127.0.0.1:55441)/" + +// TestMain points the credentials-file rung at a nonexistent path in a private temp +// dir so tests never read the developer's real ~/.config/beads/credentials. +// go-sql-driver reads no ambient env/option files, so there is nothing else to +// neutralize. Tests that exercise the file rung override it per-test with t.Setenv. +func TestMain(m *testing.M) { + dir, err := os.MkdirTemp("", "beads-mysql-cred-test") + if err != nil { + panic(err) + } + os.Setenv("BEADS_CREDENTIALS_FILE", filepath.Join(dir, "none")) + code := m.Run() + _ = os.RemoveAll(dir) + os.Exit(code) +} + +// password parses out and returns the password go-sql-driver sees. +func password(t *testing.T, dsn string) string { + t.Helper() + cfg, err := gomysql.ParseDSN(dsn) + if err != nil { + t.Fatalf("gomysql.ParseDSN(%q): %v", dsn, err) + } + return cfg.Passwd +} + +// user parses out and returns the username go-sql-driver sees. +func user(t *testing.T, dsn string) string { + t.Helper() + cfg, err := gomysql.ParseDSN(dsn) + if err != nil { + t.Fatalf("gomysql.ParseDSN(%q): %v", dsn, err) + } + return cfg.User +} + +// writeCredsFile writes an INI credentials file and returns its path. +func writeCredsFile(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "credentials") + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func TestResolveDSNCredentialCommand(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "printf cmd-pw") + t.Setenv("BEADS_MYSQL_PASSWORD", "") + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "cmd-pw" { + t.Fatalf("password = %q, want cmd-pw", pw) + } +} + +func TestResolveDSNCredentialEnv(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") + t.Setenv("BEADS_MYSQL_PASSWORD", "env-pw") + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "env-pw" { + t.Fatalf("password = %q, want env-pw", pw) + } +} + +// The command out-ranks the static env password. +func TestResolveDSNCredentialCommandBeatsEnv(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "printf cmd-wins") + t.Setenv("BEADS_MYSQL_PASSWORD", "env-loses") + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "cmd-wins" { + t.Fatalf("password = %q, want cmd-wins", pw) + } +} + +// A configured-but-erroring command aborts the open and does NOT fall through to the +// env password. Here a bare token with whitespace is rejected by parseCredential β€” a +// configured error, not a command exit failure. +func TestResolveDSNCredentialFailsClosed(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", `printf 'access denied'`) + t.Setenv("BEADS_MYSQL_PASSWORD", "env-would-be-wrong") + _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err == nil { + t.Fatal("expected an error (fail-closed); a broken command must not fall through to BEADS_MYSQL_PASSWORD") + } +} + +// A command that exits non-zero also aborts (the other configured-error shape). +func TestResolveDSNCredentialFailsClosedOnExit(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "false") + t.Setenv("BEADS_MYSQL_PASSWORD", "env-would-be-wrong") + _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err == nil { + t.Fatal("expected an error when the command exits non-zero") + } +} + +// A getToken/ExecCredential JSON envelope resolves the password end-to-end. +func TestResolveDSNCredentialJSONEnvelope(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", `printf '{"access_token":"tok-pw","expires_in":90}'`) + t.Setenv("BEADS_MYSQL_PASSWORD", "") + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "tok-pw" { + t.Fatalf("password = %q, want tok-pw", pw) + } +} + +// End-to-end: resolving a password into a userless base DSN must fail loudly (the +// grammar would silently drop it), not connect passwordless. +func TestResolveDSNCredentialUserlessDSNFailsLoudly(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") + t.Setenv("BEADS_MYSQL_PASSWORD", "some-pw") + _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, "tcp(127.0.0.1:55441)/") + if err == nil { + t.Fatal("expected an error placing a password into a userless DSN") + } +} + +// withDatabase refuses a userless password-bearing DSN (the path that skips the +// ladder via an inline ":secret@tcp(host)/" URL) rather than silently dropping it. +func TestWithDatabaseRefusesUserlessPassword(t *testing.T) { + if _, err := withDatabase(":secret@tcp(127.0.0.1:55441)/", "db"); err == nil { + t.Fatal("expected withDatabase to refuse a userless password-bearing DSN") + } + // A normal user:pass DSN still works. + if _, err := withDatabase("bts:bts@tcp(127.0.0.1:55441)/", "db"); err != nil { + t.Fatalf("withDatabase rejected a valid DSN: %v", err) + } +} + +func TestResolveDSNCredentialNothingConfigured(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") + t.Setenv("BEADS_MYSQL_PASSWORD", "") + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if got != baseDSN { + t.Fatalf("dsn = %q, want it unchanged (%q)", got, baseDSN) + } +} + +func TestResolveDSNCredentialExistingPasswordWins(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "printf should-not-run") + t.Setenv("BEADS_MYSQL_PASSWORD", "should-not-run") + withPw := "bts:already-here@tcp(127.0.0.1:55441)/" + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, withPw) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "already-here" { + t.Fatalf("password = %q, want already-here (existing password must win)", pw) + } +} + +// The credentials file resolves the password when no command or env is set. +func TestResolveDSNCredentialFile(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") + t.Setenv("BEADS_MYSQL_PASSWORD", "") + t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:55441]\npassword=file-pw\n")) + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "file-pw" { + t.Fatalf("password = %q, want file-pw", pw) + } +} + +// The env password out-ranks the credentials file. +func TestResolveDSNCredentialEnvBeatsFile(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") + t.Setenv("BEADS_MYSQL_PASSWORD", "env-pw") + t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:55441]\npassword=file-pw\n")) + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "env-pw" { + t.Fatalf("password = %q, want env-pw (env out-ranks file)", pw) + } +} + +// A broken command fails closed even when the file could supply a valid password. +func TestResolveDSNCredentialBrokenCommandNotRescuedByFile(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "false") + t.Setenv("BEADS_MYSQL_PASSWORD", "") + t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:55441]\npassword=file-pw\n")) + if _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN); err == nil { + t.Fatal("expected fail-closed; the credentials file must not rescue a broken command") + } +} + +// A unix-socket DSN has no [host:port] endpoint, so the file rung degrades to +// not-configured (no crash), leaving the DSN untouched. +func TestResolveDSNCredentialFileUnixSocketDegrades(t *testing.T) { + t.Setenv("BEADS_MYSQL_PASSWORD_COMMAND", "") + t.Setenv("BEADS_MYSQL_PASSWORD", "") + t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:55441]\npassword=file-pw\n")) + unixDSN := "bts@unix(/tmp/mysql.sock)/db" + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, unixDSN) + if err != nil { + t.Fatal(err) + } + if got != unixDSN { + t.Fatalf("dsn = %q, want it unchanged (unix socket has no file-rung endpoint)", got) + } +} + +// A set BEADS_MYSQL_CREDENTIAL_COMMAND is a hard error, pre-empting even a +// password-bearing DSN and any lower rung. +func TestResolveDSNCredentialRejectsReservedIdentityVar(t *testing.T) { + t.Setenv("BEADS_MYSQL_CREDENTIAL_COMMAND", "get-credential") + t.Setenv("BEADS_MYSQL_PASSWORD", "would-otherwise-work") + _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, "bts:has-pw@tcp(127.0.0.1:55441)/") + if err == nil || !strings.Contains(err.Error(), "BEADS_MYSQL_CREDENTIAL_COMMAND") { + t.Fatalf("expected a reserved-var rejection naming BEADS_MYSQL_CREDENTIAL_COMMAND, got %v", err) + } +} + +// staticSource is a test Source with a fixed credential, used to exercise placement +// and the identity-refusal guard directly. +type staticSource struct{ cred creds.Credential } + +func (s staticSource) Name() string { return "test" } +func (s staticSource) Resolve(context.Context) (creds.Credential, bool, error) { + return s.cred, true, nil +} + +func TestResolveDSNCredentialRefusesIdentity(t *testing.T) { + src := staticSource{cred: creds.Credential{Value: "an-identity-token", Kind: creds.KindIdentity, Source: "test"}} + _, err := resolveDSNWithSources(context.Background(), baseDSN, src) + if err == nil || !strings.Contains(err.Error(), "identity") { + t.Fatalf("expected an identity-refusal error, got %v", err) + } +} + +// A credential carrying a username (a Vault dynamic user/password pair) places both, +// overriding the base DSN's user. +func TestResolveDSNCredentialPlacesUsername(t *testing.T) { + src := staticSource{cred: creds.Credential{Value: "pw", Username: "dyn", Kind: creds.KindSecret}} + got, err := resolveDSNWithSources(context.Background(), baseDSN, src) + if err != nil { + t.Fatal(err) + } + if u, pw := user(t, got), password(t, got); u != "dyn" || pw != "pw" { + t.Fatalf("(user,password) = (%q,%q), want (dyn,pw)", u, pw) + } +} + +// A dynamic username satisfies the userless-DSN requirement: a bare tcp(host)/ base +// plus a user/password pair is a valid configuration. +func TestResolveDSNWithSourcesUsernameSatisfiesUserlessBase(t *testing.T) { + src := staticSource{cred: creds.Credential{Value: "pw", Username: "dyn", Kind: creds.KindSecret}} + got, err := resolveDSNWithSources(context.Background(), "tcp(127.0.0.1:55441)/", src) + if err != nil { + t.Fatal(err) + } + if u, pw := user(t, got), password(t, got); u != "dyn" || pw != "pw" { + t.Fatalf("(user,password) = (%q,%q), want (dyn,pw)", u, pw) + } +} diff --git a/internal/storage/mysql/dialect.go b/internal/storage/mysql/dialect.go new file mode 100644 index 000000000..24ef3200b --- /dev/null +++ b/internal/storage/mysql/dialect.go @@ -0,0 +1,23 @@ +package mysql + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/mysqldialect" +) + +// mysqlDialect is the MySQL SQL flavor. bd's canonical SQL is already MySQL, so this +// is nearly the identity dialect: mysqldialect.Open wraps go-sql-driver with a single +// rewrite (the self-referential is_blocked recompute UPDATE that MySQL 8 rejects) and +// passes everything else through verbatim. The dsn already targets the workspace +// database. +type mysqlDialect struct { + dsn string +} + +func (d mysqlDialect) Name() string { return "mysql" } + +func (d mysqlDialect) Open(_ context.Context) (*sql.DB, error) { + return mysqldialect.Open(d.dsn) +} diff --git a/internal/storage/mysql/dsn.go b/internal/storage/mysql/dsn.go new file mode 100644 index 000000000..5097e549a --- /dev/null +++ b/internal/storage/mysql/dsn.go @@ -0,0 +1,53 @@ +package mysql + +import ( + "fmt" + + gomysql "github.com/go-sql-driver/mysql" +) + +// RedactPassword returns dsn with the password removed, or an error if it cannot be +// stripped. It uses go-sql-driver's own parser (never hand-rolled string surgery) and +// verifies with a re-parse that no password survives, failing closed. bd init persists +// the returned string to metadata.json; the password is re-supplied at open time by +// the credential ladder (see resolveDSNCredential in credential.go). +func RedactPassword(dsn string) (string, error) { + cfg, err := gomysql.ParseDSN(dsn) + if err != nil { + return "", fmt.Errorf("parse DSN: %w", err) + } + cfg.Passwd = "" + stripped := cfg.FormatDSN() + if again, err := gomysql.ParseDSN(stripped); err != nil || again.Passwd != "" { + return "", fmt.Errorf("refusing to persist a connection string that still carries a password; supply it via BEADS_MYSQL_PASSWORD instead of embedding it in --mysql-url") + } + return stripped, nil +} + +// withDatabase returns dsn with its default database set to database (or cleared when +// database is ""). Used to derive a server connection (for CREATE DATABASE) and the +// per-workspace connection from one base DSN. It also forces parseTime and UTC so +// datetime columns round-trip identically to the Dolt reference. +func withDatabase(dsn, database string) (string, error) { + cfg, err := gomysql.ParseDSN(dsn) + if err != nil { + return "", err + } + // The go-sql-driver grammar cannot carry a password without a username β€” + // FormatDSN would silently drop it. Refuse loudly rather than reformat this DSN + // into a passwordless one and connect (the same invariant mysqldialect.WithPassword + // enforces; this covers the path where a userless password-bearing DSN reaches us + // directly, e.g. an inline ":secret@tcp(host)/" URL that skips the credential ladder). + if cfg.User == "" && cfg.Passwd != "" { + return "", fmt.Errorf("connection string carries a password but no username; the MySQL DSN grammar cannot carry a password without a user (it would be silently dropped) β€” add a user to the DSN") + } + cfg.DBName = database + cfg.ParseTime = true + if cfg.Params == nil { + cfg.Params = map[string]string{} + } + // Pin the session time zone to UTC so CURRENT_TIMESTAMP defaults and reads match + // the shared layer's UTC assumption (and the Dolt reference). + cfg.Params["time_zone"] = "'+00:00'" + return cfg.FormatDSN(), nil +} diff --git a/internal/storage/mysql/fromconfig.go b/internal/storage/mysql/fromconfig.go new file mode 100644 index 000000000..4ee3d5112 --- /dev/null +++ b/internal/storage/mysql/fromconfig.go @@ -0,0 +1,57 @@ +package mysql + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage" +) + +// NewFromConfig opens the MySQL backend for a workspace. It reads the base server DSN +// and per-workspace database from .beads/metadata.json, resolves the password through +// the credential ladder (command > env > credentials file, fail-closed) and places it into the DSN β€” +// never persisting it β€” then provisions and returns the store. It is the factory arm +// cmd/bd dispatches to when metadata.json has backend="mysql". +func NewFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { + cfg, err := configfile.Load(beadsDir) + if err != nil { + return nil, fmt.Errorf("mysql: load config: %w", err) + } + dsn := cfg.GetMySQLDSN() + if dsn == "" { + return nil, fmt.Errorf("mysql: no DSN (set mysql_dsn in metadata.json, or BEADS_MYSQL_URL)") + } + dsn, err = resolveDSNCredential(ctx, cfg, dsn) + if err != nil { + return nil, fmt.Errorf("mysql: resolve credential: %w", err) + } + database := cfg.GetMySQLDatabase() + if database == "" { + return nil, fmt.Errorf("mysql: no database (set mysql_database in metadata.json)") + } + return Provision(ctx, dsn, database) +} + +// Provision opens the MySQL backend from an explicit server DSN and workspace +// database: it creates the database if absent and applies the schema (InitSchema β€” +// idempotent, config seeds on first provision only), then returns the store over the +// per-workspace connection. baseDSN may carry a password; it is used only to connect +// and is never persisted. bd init calls this directly; NewFromConfig calls it after +// resolving DSN/database from metadata. +func Provision(ctx context.Context, baseDSN, database string) (storage.DoltStorage, error) { + if baseDSN == "" { + return nil, fmt.Errorf("mysql: empty DSN") + } + if database == "" { + return nil, fmt.Errorf("mysql: empty database") + } + if err := InitSchema(ctx, baseDSN, database); err != nil { + return nil, err + } + wsDSN, err := withDatabase(baseDSN, database) + if err != nil { + return nil, fmt.Errorf("mysql: parse DSN: %w", err) + } + return New(ctx, Config{DSN: wsDSN}) +} diff --git a/internal/storage/mysql/redact_test.go b/internal/storage/mysql/redact_test.go new file mode 100644 index 000000000..4bd55a76e --- /dev/null +++ b/internal/storage/mysql/redact_test.go @@ -0,0 +1,117 @@ +package mysql + +import ( + "testing" + + gomysql "github.com/go-sql-driver/mysql" +) + +// placeholderPW is an obvious non-secret stand-in used wherever a password appears in +// these tests, so nothing here can be mistaken for a leaked real credential. +const placeholderPW = "s3cr3t-PLACEHOLDER" + +// TestRedactPassword locks the security invariant for mysql.RedactPassword across every +// secret-bearing DSN shape the go-sql-driver grammar accepts. For each case it asserts +// BOTH: +// +// (a) the persisted string, re-parsed by go-sql-driver β€” the SAME parser that opens +// the connection β€” carries NO password (prove the secret is gone); and +// (b) every non-secret component (user, host:port, network, database, params, and the +// parseTime flag) survives redaction. +// +// A no-password DSN must additionally round-trip byte-for-byte unchanged. +func TestRedactPassword(t *testing.T) { + cases := []struct { + name string + dsn string + wantUser string + wantAddr string + wantNet string + wantDB string + wantParseTime bool + wantParams map[string]string + roundTripSame bool + }{ + { + name: "password present", + dsn: "bts:" + placeholderPW + "@tcp(127.0.0.1:3306)/beadsdb", + wantUser: "bts", wantAddr: "127.0.0.1:3306", wantNet: "tcp", wantDB: "beadsdb", + }, + { + // go-sql-driver splits the userinfo password on the LAST '@', so a password + // full of ':' '/' '#' '@' must still be stripped cleanly. + name: "password with special chars", + dsn: "bts:p@ss:w/rd#-" + placeholderPW + "@tcp(127.0.0.1:3306)/beadsdb", + wantUser: "bts", wantAddr: "127.0.0.1:3306", wantNet: "tcp", wantDB: "beadsdb", + }, + { + name: "password present, params + parseTime preserved", + dsn: "bts:" + placeholderPW + "@tcp(db.corp:3307)/beadsdb?parseTime=true&myapp=beads", + wantUser: "bts", wantAddr: "db.corp:3307", wantNet: "tcp", wantDB: "beadsdb", + wantParseTime: true, wantParams: map[string]string{"myapp": "beads"}, + }, + { + // The go-sql-driver grammar cannot carry a password without a username; + // FormatDSN drops the whole user:pass@ block, so redaction still removes the + // secret. (The loud refusal for this shape lives in withDatabase/WithPassword, + // not RedactPassword β€” here the invariant is only that the password is gone.) + name: "password but no username", + dsn: ":" + placeholderPW + "@tcp(127.0.0.1:3306)/beadsdb", + wantUser: "", wantAddr: "127.0.0.1:3306", wantNet: "tcp", wantDB: "beadsdb", + }, + { + // A canonical, param-free DSN must survive redaction byte-for-byte + // (go-sql-driver reorders query params on reformat, so the round-trip + // invariant is asserted only against a DSN already in canonical form). + name: "no password round-trips unchanged", + dsn: "bts@tcp(127.0.0.1:3306)/beadsdb", + wantUser: "bts", wantAddr: "127.0.0.1:3306", wantNet: "tcp", wantDB: "beadsdb", + roundTripSame: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := RedactPassword(tc.dsn) + if err != nil { + t.Fatalf("RedactPassword(%q) errored: %v", tc.dsn, err) + } + + // (a) The authoritative security check: the same parser go-sql-driver uses to + // open the connection must find no password in the persisted string. + if pw := password(t, got); pw != "" { + t.Fatalf("PASSWORD LEAK: RedactPassword(%q) = %q still carries password %q", tc.dsn, got, pw) + } + + // (b) Non-secret components must survive. + cfg, err := gomysql.ParseDSN(got) + if err != nil { + t.Fatalf("redacted DSN no longer parses: %q: %v", got, err) + } + if cfg.User != tc.wantUser { + t.Errorf("user = %q, want %q", cfg.User, tc.wantUser) + } + if cfg.Addr != tc.wantAddr { + t.Errorf("addr = %q, want %q", cfg.Addr, tc.wantAddr) + } + if cfg.Net != tc.wantNet { + t.Errorf("net = %q, want %q", cfg.Net, tc.wantNet) + } + if cfg.DBName != tc.wantDB { + t.Errorf("dbname = %q, want %q", cfg.DBName, tc.wantDB) + } + if cfg.ParseTime != tc.wantParseTime { + t.Errorf("parseTime = %v, want %v (must be preserved)", cfg.ParseTime, tc.wantParseTime) + } + for k, want := range tc.wantParams { + if got := cfg.Params[k]; got != want { + t.Errorf("param %q = %q, want %q (must be preserved)", k, got, want) + } + } + + if tc.roundTripSame && got != tc.dsn { + t.Errorf("no-password DSN must round-trip unchanged:\n in = %q\n out = %q", tc.dsn, got) + } + }) + } +} diff --git a/internal/storage/mysql/schema.go b/internal/storage/mysql/schema.go new file mode 100644 index 000000000..d178b02c9 --- /dev/null +++ b/internal/storage/mysql/schema.go @@ -0,0 +1,162 @@ +package mysql + +import ( + "context" + "database/sql" + _ "embed" + "errors" + "fmt" + "regexp" + "strings" +) + +// ddl is the MySQL backend schema, embedded from schema.sql. It is the Dolt +// `SHOW CREATE TABLE` output at HEAD (all migrations applied) with the handful of +// Dolt-vs-MySQL-8 fixes required by stock MySQL: TEXT/JSON literal defaults β†’ +// expression form `DEFAULT (”)`, the overlong `spec_id` index β†’ a 191-char prefix +// (InnoDB's 3072-byte key cap), and the one-target CHECK constraints dropped (MySQL 8 +// forbids a column in both a CHECK and an FK cascade; issueops enforces the invariant +// in code). No data-plane translation is needed β€” bd's canonical SQL is MySQL. +// +//go:embed schema.sql +var ddl string + +const ( + schemaVersion = "1" + schemaVersionKey = "mysql_schema_version" +) + +// identRe restricts a database name to a plain identifier so it can be safely +// interpolated into CREATE DATABASE (which cannot take a placeholder). +var identRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// InitSchema creates the workspace database if absent (via a server connection with +// no default database), applies the DDL to it, seeds the default config rows on first +// provision, and stamps the schema version. baseDSN is the server DSN; database is the +// per-workspace database (MySQL's unit of isolation, analogous to a Postgres schema). +func InitSchema(ctx context.Context, baseDSN, database string) error { + if !identRe.MatchString(database) { + return fmt.Errorf("mysql: invalid database name %q", database) + } + serverDSN, err := withDatabase(baseDSN, "") + if err != nil { + return fmt.Errorf("mysql: parse DSN: %w", err) + } + srv, err := sql.Open("mysql", serverDSN) + if err != nil { + return fmt.Errorf("mysql: open server: %w", err) + } + if _, err := srv.ExecContext(ctx, "CREATE DATABASE IF NOT EXISTS `"+database+"` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin"); err != nil { + _ = srv.Close() + return fmt.Errorf("mysql: create database %q: %w", database, err) + } + _ = srv.Close() + + wsDSN, err := withDatabase(baseDSN, database) + if err != nil { + return fmt.Errorf("mysql: parse DSN: %w", err) + } + db, err := sql.Open("mysql", wsDSN) + if err != nil { + return fmt.Errorf("mysql: open workspace db: %w", err) + } + defer db.Close() + for _, stmt := range splitStatements(ddl) { + if _, err := db.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("mysql: exec DDL statement: %w\n-- statement:\n%s", err, stmt) + } + } + fresh, err := stampSchemaVersion(ctx, db) + if err != nil { + return err + } + if fresh { + if err := seedDefaultConfig(ctx, db); err != nil { + return err + } + } + return nil +} + +// defaultConfigSeeds mirrors migration 0016 exactly β€” the config rows a fresh Dolt +// workspace materializes on init. Seeded only on first provision (like the one-shot +// migration), so a `config unset` is never resurrected on the next open. +var defaultConfigSeeds = [][2]string{ + {"compaction_enabled", "false"}, + {"compact_tier1_days", "30"}, + {"compact_tier1_dep_levels", "2"}, + {"compact_tier2_days", "90"}, + {"compact_tier2_dep_levels", "5"}, + {"compact_tier2_commits", "100"}, + {"compact_batch_size", "50"}, + {"compact_parallel_workers", "5"}, + {"auto_compact_enabled", "false"}, +} + +func seedDefaultConfig(ctx context.Context, db *sql.DB) error { + for _, kv := range defaultConfigSeeds { + if _, err := db.ExecContext(ctx, + "INSERT IGNORE INTO config (`key`, `value`) VALUES (?, ?)", kv[0], kv[1]); err != nil { + return fmt.Errorf("mysql: seed default config %q: %w", kv[0], err) + } + } + return nil +} + +// stampSchemaVersion records schemaVersion in the metadata table on first provision +// and refuses to open a workspace written by a binary with a different schema version +// (the proof-wedge has no migrator). Returns fresh=true only on the first stamp. +func stampSchemaVersion(ctx context.Context, db *sql.DB) (fresh bool, err error) { + var stored string + err = db.QueryRowContext(ctx, "SELECT `value` FROM metadata WHERE `key` = ?", schemaVersionKey).Scan(&stored) + switch { + case errors.Is(err, sql.ErrNoRows): + // Provision runs on every open; concurrent first opens of the same + // workspace database can both reach here. INSERT IGNORE lets both + // succeed; only the opener that inserted the row (RowsAffected==1) is + // fresh and seeds defaults (seedDefaultConfig is already INSERT IGNORE). + res, ierr := db.ExecContext(ctx, "INSERT IGNORE INTO metadata (`key`, `value`) VALUES (?, ?)", schemaVersionKey, schemaVersion) + if ierr != nil { + return false, fmt.Errorf("mysql: stamp schema version: %w", ierr) + } + if n, _ := res.RowsAffected(); n == 1 { + return true, nil + } + // Lost the race: re-read what the winner stored and version-check it. + if rerr := db.QueryRowContext(ctx, "SELECT `value` FROM metadata WHERE `key` = ?", schemaVersionKey).Scan(&stored); rerr != nil { + return false, fmt.Errorf("mysql: read schema version after conflict: %w", rerr) + } + if stored != schemaVersion { + return false, fmt.Errorf("mysql: workspace schema version %s, this binary requires %s β€” no migrator in the proof-wedge, recreate the workspace or use a matching binary", stored, schemaVersion) + } + return false, nil + case err != nil: + return false, fmt.Errorf("mysql: read schema version: %w", err) + case stored != schemaVersion: + return false, fmt.Errorf("mysql: workspace schema version %s, this binary requires %s β€” no migrator in the proof-wedge, recreate the workspace or use a matching binary", stored, schemaVersion) + default: + return false, nil + } +} + +// splitStatements splits the DDL on ';' at statement boundaries. The MySQL schema has +// no procedural bodies (triggers/procedures) or embedded ';' β€” the CHECK constraints +// that would complicate this were removed β€” so a plain split suffices. Blank fragments +// (and SQL line comments) are dropped. +func splitStatements(ddl string) []string { + var stmts []string + for _, raw := range strings.Split(ddl, ";") { + var b strings.Builder + for _, line := range strings.Split(raw, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "--") { + continue + } + b.WriteString(line) + b.WriteString("\n") + } + if s := strings.TrimSpace(b.String()); s != "" { + stmts = append(stmts, s) + } + } + return stmts +} diff --git a/internal/storage/mysql/schema.sql b/internal/storage/mysql/schema.sql new file mode 100644 index 000000000..a692ec906 --- /dev/null +++ b/internal/storage/mysql/schema.sql @@ -0,0 +1,337 @@ +CREATE TABLE IF NOT EXISTS `issues` ( + `id` varchar(255) NOT NULL, + `content_hash` varchar(64), + `title` varchar(500) NOT NULL, + `description` longtext NOT NULL, + `design` longtext NOT NULL, + `acceptance_criteria` longtext NOT NULL, + `notes` longtext NOT NULL, + `status` varchar(32) NOT NULL DEFAULT 'open', + `priority` int NOT NULL DEFAULT '2', + `issue_type` varchar(32) NOT NULL DEFAULT 'task', + `assignee` varchar(255), + `estimated_minutes` int, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `created_by` varchar(255) DEFAULT '', + `owner` varchar(255) DEFAULT '', + `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `closed_at` datetime, + `closed_by_session` varchar(255) DEFAULT '', + `external_ref` varchar(255), + `spec_id` varchar(1024), + `compaction_level` int DEFAULT '0', + `compacted_at` datetime, + `compacted_at_commit` varchar(64), + `original_size` int, + `sender` varchar(255) DEFAULT '', + `ephemeral` tinyint(1) DEFAULT '0', + `wisp_type` varchar(32) DEFAULT '', + `pinned` tinyint(1) DEFAULT '0', + `is_template` tinyint(1) DEFAULT '0', + `mol_type` varchar(32) DEFAULT '', + `work_type` varchar(32) DEFAULT 'mutex', + `source_system` varchar(255) DEFAULT '', + `metadata` json DEFAULT (json_object()), + `source_repo` varchar(512) DEFAULT '', + `close_reason` longtext DEFAULT (''), + `event_kind` varchar(32) DEFAULT '', + `actor` varchar(255) DEFAULT '', + `target` varchar(255) DEFAULT '', + `payload` text DEFAULT (''), + `await_type` varchar(32) DEFAULT '', + `await_id` varchar(255) DEFAULT '', + `timeout_ns` bigint DEFAULT '0', + `waiters` text DEFAULT (''), + `hook_bead` varchar(255) DEFAULT '', + `role_bead` varchar(255) DEFAULT '', + `agent_state` varchar(32) DEFAULT '', + `last_activity` datetime, + `role_type` varchar(32) DEFAULT '', + `rig` varchar(255) DEFAULT '', + `due_at` datetime, + `defer_until` datetime, + `no_history` tinyint(1) DEFAULT '0', + `started_at` datetime, + `is_blocked` tinyint(1) NOT NULL DEFAULT '0', + `lease_expires_at` datetime, + `heartbeat_at` datetime, + `row_lock` bigint NOT NULL DEFAULT 0, + PRIMARY KEY (`id`), + KEY `idx_issues_assignee` (`assignee`), + KEY `idx_issues_created_at` (`created_at`), + KEY `idx_issues_defer_until` (`defer_until`), + KEY `idx_issues_external_ref` (`external_ref`), + KEY `idx_issues_is_blocked` (`is_blocked`,`status`), + KEY `idx_issues_issue_type` (`issue_type`), + KEY `idx_issues_priority` (`priority`), + KEY `idx_issues_spec_id` (`spec_id`(191)), + KEY `idx_issues_status_updated_at` (`status`,`updated_at`), + KEY `idx_issues_lease` (`status`,`lease_expires_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `wisps` ( + `id` varchar(255) NOT NULL, + `content_hash` varchar(64), + `title` varchar(500) NOT NULL, + `description` longtext NOT NULL DEFAULT (''), + `design` longtext NOT NULL DEFAULT (''), + `acceptance_criteria` longtext NOT NULL DEFAULT (''), + `notes` longtext NOT NULL DEFAULT (''), + `status` varchar(32) NOT NULL DEFAULT 'open', + `priority` int NOT NULL DEFAULT '2', + `issue_type` varchar(32) NOT NULL DEFAULT 'task', + `assignee` varchar(255), + `estimated_minutes` int, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `created_by` varchar(255) DEFAULT '', + `owner` varchar(255) DEFAULT '', + `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `closed_at` datetime, + `closed_by_session` varchar(255) DEFAULT '', + `external_ref` varchar(255), + `spec_id` varchar(1024), + `compaction_level` int DEFAULT '0', + `compacted_at` datetime, + `compacted_at_commit` varchar(64), + `original_size` int, + `sender` varchar(255) DEFAULT '', + `ephemeral` tinyint(1) DEFAULT '0', + `wisp_type` varchar(32) DEFAULT '', + `pinned` tinyint(1) DEFAULT '0', + `is_template` tinyint(1) DEFAULT '0', + `mol_type` varchar(32) DEFAULT '', + `work_type` varchar(32) DEFAULT 'mutex', + `source_system` varchar(255) DEFAULT '', + `metadata` json DEFAULT (json_object()), + `source_repo` varchar(512) DEFAULT '', + `close_reason` longtext DEFAULT (''), + `event_kind` varchar(32) DEFAULT '', + `actor` varchar(255) DEFAULT '', + `target` varchar(255) DEFAULT '', + `payload` text DEFAULT (''), + `await_type` varchar(32) DEFAULT '', + `await_id` varchar(255) DEFAULT '', + `timeout_ns` bigint DEFAULT '0', + `waiters` text DEFAULT (''), + `hook_bead` varchar(255) DEFAULT '', + `role_bead` varchar(255) DEFAULT '', + `agent_state` varchar(32) DEFAULT '', + `last_activity` datetime, + `role_type` varchar(32) DEFAULT '', + `rig` varchar(255) DEFAULT '', + `due_at` datetime, + `defer_until` datetime, + `no_history` tinyint(1) DEFAULT '0', + `started_at` datetime, + `is_blocked` tinyint(1) NOT NULL DEFAULT '0', + `lease_expires_at` datetime, + `heartbeat_at` datetime, + `row_lock` bigint NOT NULL DEFAULT 0, + PRIMARY KEY (`id`), + KEY `idx_wisps_assignee` (`assignee`), + KEY `idx_wisps_created_at` (`created_at`), + KEY `idx_wisps_external_ref` (`external_ref`), + KEY `idx_wisps_is_blocked` (`is_blocked`,`status`), + KEY `idx_wisps_issue_type` (`issue_type`), + KEY `idx_wisps_priority` (`priority`), + KEY `idx_wisps_spec_id` (`spec_id`(191)), + KEY `idx_wisps_status` (`status`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `labels` ( + `issue_id` varchar(255) NOT NULL, + `label` varchar(255) NOT NULL, + PRIMARY KEY (`issue_id`,`label`), + KEY `idx_labels_label` (`label`), + CONSTRAINT `fk_labels_issue` FOREIGN KEY (`issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `wisp_labels` ( + `issue_id` varchar(255) NOT NULL, + `label` varchar(255) NOT NULL, + PRIMARY KEY (`issue_id`,`label`), + KEY `idx_wisp_labels_label` (`label`), + CONSTRAINT `fk_wisp_labels_issue` FOREIGN KEY (`issue_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `dependencies` ( + `id` char(36) NOT NULL, + `issue_id` varchar(255) NOT NULL, + `type` varchar(32) NOT NULL DEFAULT 'blocks', + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `created_by` varchar(255) NOT NULL, + `metadata` json DEFAULT (json_object()), + `thread_id` varchar(255) DEFAULT '', + `depends_on_issue_id` varchar(255), + `depends_on_wisp_id` varchar(255), + `depends_on_external` varchar(255), + PRIMARY KEY (`id`), + KEY `idx_dep_external_target` (`depends_on_external`), + KEY `idx_dep_issue_target` (`depends_on_issue_id`), + KEY `idx_dep_type_external` (`type`,`depends_on_external`), + KEY `idx_dep_type_issue` (`type`,`depends_on_issue_id`), + KEY `idx_dep_type_wisp` (`type`,`depends_on_wisp_id`), + KEY `idx_dep_wisp_target` (`depends_on_wisp_id`), + KEY `idx_dependencies_issue` (`issue_id`), + KEY `idx_dependencies_thread` (`thread_id`), + UNIQUE KEY `uk_dep_external_target` (`issue_id`,`depends_on_external`), + UNIQUE KEY `uk_dep_issue_target` (`issue_id`,`depends_on_issue_id`), + UNIQUE KEY `uk_dep_wisp_target` (`issue_id`,`depends_on_wisp_id`), + CONSTRAINT `fk_dep_issue` FOREIGN KEY (`issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `fk_dep_issue_target` FOREIGN KEY (`depends_on_issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `wisp_dependencies` ( + `id` char(36) NOT NULL, + `issue_id` varchar(255) NOT NULL, + `depends_on_issue_id` varchar(255), + `depends_on_wisp_id` varchar(255), + `depends_on_external` varchar(255), + `type` varchar(32) NOT NULL DEFAULT 'blocks', + `created_at` datetime DEFAULT CURRENT_TIMESTAMP, + `created_by` varchar(255) DEFAULT '', + `metadata` json DEFAULT (json_object()), + `thread_id` varchar(255) DEFAULT '', + PRIMARY KEY (`id`), + KEY `fk_wisp_dep_issue_target` (`depends_on_issue_id`), + KEY `fk_wisp_dep_wisp_target` (`depends_on_wisp_id`), + KEY `idx_wisp_dep_type` (`type`), + KEY `idx_wisp_dep_type_external` (`type`,`depends_on_external`), + KEY `idx_wisp_dep_type_issue` (`type`,`depends_on_issue_id`), + KEY `idx_wisp_dep_type_wisp` (`type`,`depends_on_wisp_id`), + UNIQUE KEY `uk_wisp_dep_external_target` (`issue_id`,`depends_on_external`), + UNIQUE KEY `uk_wisp_dep_issue_target` (`issue_id`,`depends_on_issue_id`), + UNIQUE KEY `uk_wisp_dep_wisp_target` (`issue_id`,`depends_on_wisp_id`), + CONSTRAINT `fk_wisp_dep_issue` FOREIGN KEY (`issue_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `fk_wisp_dep_issue_target` FOREIGN KEY (`depends_on_issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `fk_wisp_dep_wisp_target` FOREIGN KEY (`depends_on_wisp_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `comments` ( + `id` char(36) NOT NULL, + `issue_id` varchar(255) NOT NULL, + `author` varchar(255) NOT NULL, + `text` longtext NOT NULL, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_comments_created_at` (`created_at`), + KEY `idx_comments_issue` (`issue_id`), + CONSTRAINT `fk_comments_issue` FOREIGN KEY (`issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `wisp_comments` ( + `id` char(36) NOT NULL, + `issue_id` varchar(255) NOT NULL, + `author` varchar(255) DEFAULT '', + `text` text NOT NULL, + `created_at` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_wisp_comments_issue` (`issue_id`), + CONSTRAINT `fk_wisp_comments_issue` FOREIGN KEY (`issue_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `events` ( + `id` char(36) NOT NULL, + `issue_id` varchar(255) NOT NULL, + `event_type` varchar(32) NOT NULL, + `actor` varchar(255) NOT NULL, + `old_value` longtext, + `new_value` longtext, + `comment` text, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_events_created_at` (`created_at`), + KEY `idx_events_issue` (`issue_id`), + CONSTRAINT `fk_events_issue` FOREIGN KEY (`issue_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `wisp_events` ( + `id` char(36) NOT NULL, + `issue_id` varchar(255) NOT NULL, + `event_type` varchar(32) NOT NULL, + `actor` varchar(255) DEFAULT '', + `old_value` longtext DEFAULT (''), + `new_value` longtext DEFAULT (''), + `comment` text DEFAULT (''), + `created_at` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_wisp_events_created_at` (`created_at`), + KEY `idx_wisp_events_issue` (`issue_id`), + CONSTRAINT `fk_wisp_events_issue` FOREIGN KEY (`issue_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `config` ( + `key` varchar(255) NOT NULL, + `value` text NOT NULL, + PRIMARY KEY (`key`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `metadata` ( + `key` varchar(255) NOT NULL, + `value` text NOT NULL, + PRIMARY KEY (`key`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `local_metadata` ( + `key` varchar(255) NOT NULL, + `value` text NOT NULL DEFAULT (''), + PRIMARY KEY (`key`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `issue_counter` ( + `prefix` varchar(255) NOT NULL, + `last_id` int NOT NULL DEFAULT '0', + PRIMARY KEY (`prefix`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `child_counters` ( + `parent_id` varchar(255) NOT NULL, + `last_child` int NOT NULL DEFAULT '0', + PRIMARY KEY (`parent_id`), + CONSTRAINT `fk_counter_parent` FOREIGN KEY (`parent_id`) REFERENCES `issues` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `wisp_child_counters` ( + `parent_id` varchar(255) NOT NULL, + `last_child` int NOT NULL DEFAULT '0', + PRIMARY KEY (`parent_id`), + CONSTRAINT `fk_wisp_child_counters_parent` FOREIGN KEY (`parent_id`) REFERENCES `wisps` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `custom_statuses` ( + `name` varchar(64) NOT NULL, + `category` varchar(32) NOT NULL DEFAULT 'unspecified', + PRIMARY KEY (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `custom_types` ( + `name` varchar(64) NOT NULL, + PRIMARY KEY (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; + + +CREATE TABLE IF NOT EXISTS `repo_mtimes` ( + `repo_path` varchar(512) NOT NULL, + `jsonl_path` varchar(512) NOT NULL, + `mtime_ns` bigint NOT NULL, + `last_checked` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`repo_path`), + KEY `idx_repo_mtimes_checked` (`last_checked`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin; diff --git a/internal/storage/mysql/seed_test.go b/internal/storage/mysql/seed_test.go new file mode 100644 index 000000000..6a6f8353b --- /dev/null +++ b/internal/storage/mysql/seed_test.go @@ -0,0 +1,54 @@ +package mysql + +import ( + "context" + "database/sql" + "fmt" + "os" + "testing" + "time" +) + +// TestSeedOnlyOnFirstProvision guards the same regression the Postgres backend has: +// InitSchema runs on every open, so seeding default config unconditionally would +// resurrect a key the user `config unset`. Seeding must be gated on first provision. +func TestSeedOnlyOnFirstProvision(t *testing.T) { + url := os.Getenv("BEADS_MYSQL_TEST_URL") + if url == "" { + t.Skip("BEADS_MYSQL_TEST_URL not set") + } + database := fmt.Sprintf("seedonce_%d", time.Now().UnixNano()) + ctx := context.Background() + t.Cleanup(func() { + if serverDSN, e := withDatabase(url, ""); e == nil { + if srv, e2 := sql.Open("mysql", serverDSN); e2 == nil { + _, _ = srv.ExecContext(context.Background(), "DROP DATABASE IF EXISTS `"+database+"`") + _ = srv.Close() + } + } + }) + + st, err := Provision(ctx, url, database) + if err != nil { + t.Fatalf("Provision: %v", err) + } + if v, _ := st.GetConfig(ctx, "auto_compact_enabled"); v != "false" { + t.Fatalf("first provision: want seeded auto_compact_enabled=false, got %q", v) + } + if err := st.DeleteConfig(ctx, "auto_compact_enabled"); err != nil { + t.Fatalf("DeleteConfig: %v", err) + } + if v, _ := st.GetConfig(ctx, "auto_compact_enabled"); v != "" { + t.Fatalf("after unset: want empty, got %q", v) + } + _ = st.Close() + + st2, err := Provision(ctx, url, database) + if err != nil { + t.Fatalf("re-Provision: %v", err) + } + defer st2.Close() + if v, _ := st2.GetConfig(ctx, "auto_compact_enabled"); v != "" { + t.Fatalf("REGRESSION: re-provision resurrected an unset key: auto_compact_enabled=%q (want empty)", v) + } +} diff --git a/internal/storage/mysql/store.go b/internal/storage/mysql/store.go new file mode 100644 index 000000000..14cd7d325 --- /dev/null +++ b/internal/storage/mysql/store.go @@ -0,0 +1,69 @@ +// Package mysql is the MySQL backend for bd: the shared sqlkit.Store bundled with +// the identity MySQL dialect and a no-op readiness strategy, plus the non-Dolt +// markers. bd's canonical SQL is MySQL-dialect, so unlike Postgres there is no +// translating driver β€” the only backend-specific work is a curated MySQL-8 schema +// (Dolt is more permissive than stock MySQL 8 at the DDL level). +package mysql + +import ( + "context" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/sqlkit" +) + +// Backend is the metadata.json backend identifier for this store. +const Backend = "mysql" + +// Store is the MySQL backend. It embeds *sqlkit.Store (inheriting every core method) +// and the generated typed-unsupported shell for the non-core surface. +type Store struct { + *sqlkit.Store + unsupportedDoltStorage +} + +// Compile-time guard on the generated skip list: the composite backend (real +// *sqlkit.Store methods plus the typed-unsupported shell) must fully satisfy +// storage.DoltStorage. This fails the build on skip-list drift β€” a skipped +// method sqlkit does not actually implement (missing method), or a method both +// implemented and stubbed (ambiguous selector) β€” so no method can silently +// become "unsupported". +var _ storage.DoltStorage = (*Store)(nil) + +// Config configures a MySQL backend. DSN targets the per-workspace database; the +// password is merged from env at open and never persisted to metadata.json. +type Config struct { + DSN string +} + +// New builds the MySQL backend from its strategy bundle: the identity mysql dialect +// and a no-op readiness strategy (issueops already maintains is_blocked in-tx, exactly +// like the Dolt reference). +func New(ctx context.Context, cfg Config) (*Store, error) { + base, err := sqlkit.New(ctx, sqlkit.Config{ + Dialect: mysqlDialect{dsn: cfg.DSN}, + Readiness: sqlkit.NoopReadiness{}, + }) + if err != nil { + return nil, err + } + return &Store{Store: base}, nil +} + +// CommitGraphUnsupported tells cmd/bd's PostRun to skip Dolt-only maintenance. +func (s *Store) CommitGraphUnsupported() bool { return true } + +// CommitPending is a no-op on a non-version-controlled backend. +func (s *Store) CommitPending(_ context.Context, _ string) (bool, error) { return false, nil } + +// Commit and its variants are no-ops on MySQL, not unsupported: every sqlkit write +// already committed its own SQL transaction, so the data is durable. Command paths +// call store.Commit() opportunistically; returning nil lets those writes stand. +func (s *Store) Commit(_ context.Context, _ string) error { return nil } +func (s *Store) CommitWithConfig(_ context.Context, _ string) error { return nil } +func (s *Store) CommitMergeResolution(_ context.Context, _ string) error { return nil } + +var ( + _ storage.DoltStorage = (*Store)(nil) + _ storage.NonCommitGraphBackend = (*Store)(nil) +) diff --git a/internal/storage/mysql/unsupported.go b/internal/storage/mysql/unsupported.go new file mode 100644 index 000000000..6fe9487c0 --- /dev/null +++ b/internal/storage/mysql/unsupported.go @@ -0,0 +1,17 @@ +package mysql + +import ( + "github.com/steveyegge/beads/internal/storage" +) + +// Regeneration mirrors postgres: the mysql shell is the exact complement of +// *sqlkit.Store's method set (identical base + identical Commit/CommitGraph overrides, +// so the complement is identical to postgres's shell modulo package name). +// +//go:generate go run ../unsupportedgen -pkg mysql -src .. -out unsupported_gen.go -type DoltStorage -skip AddDependency,AddIssueComment,AddLabel,ClaimIssue,ClaimReadyIssue,Close,CloseIssue,Commit,CommitMergeResolution,CommitPending,CommitWithConfig,CountDependencies,CountDependents,CountEvents,CountIssueComments,CountIssues,CountIssuesByGroup,CreateIssue,CreateIssues,DeleteConfig,DeleteIssue,DeleteIssues,DetectCycles,GetAllConfig,GetBlockedIssues,GetBlockingInfoForIssues,GetCommentCounts,GetCommentsForIssues,GetConfig,GetCustomStatuses,GetCustomStatusesDetailed,GetCustomTypes,GetDependencies,GetDependenciesWithMetadata,GetDependencyCounts,GetDependencyRecords,GetDependencyRecordsForIssues,GetDependencyTree,GetDependents,GetDependentsWithMetadata,GetEpicsEligibleForClosure,GetEvents,GetInfraTypes,GetIssue,GetIssueByExternalRef,GetIssueComments,GetIssuesByIDs,GetIssuesByLabel,GetLabels,GetLabelsForIssues,GetLocalMetadata,GetMetadata,GetNewlyUnblockedByClose,GetNextChildID,GetReadyWork,GetReadyWorkWithCounts,GetStaleIssues,GetStatistics,IsBlocked,IsInfraTypeCtx,IterBlockedIssues,IterDependenciesWithMetadata,IterDependentsWithMetadata,IterEvents,IterIssueComments,IterIssues,IterReadyWork,IterWisps,ListWisps,RemoveDependency,RemoveLabel,ReopenIssue,RunInTransaction,SearchIssues,SearchIssuesWithCounts,SetConfig,SetLocalMetadata,SetMetadata,UpdateIssue,UpdateIssueType,AddComment,ClearRepoMtime,CountDependentsByStatus,DeleteIssuesBySourceRepo,FindWispDependentsRecursive,GetAllDependencyRecords,GetAllEventsSince,GetMoleculeLastActivity,GetMoleculeProgress,GetRepoMtime,ImportIssueComment,IterAllDependencyRecords,IterAllEventsSince,PromoteFromEphemeral,SetRepoMtime,UpdateIssueID,CreateIssuesWithFullOptions,SlotClear,SlotGet,SlotSet,SearchIssueIDs,HeartbeatIssue,ReclaimExpiredLeases,UnclaimIssue + +// errUnsupported is the constructor every generated stub in unsupported_gen.go calls. +// It returns the backend-agnostic *storage.ErrUnsupported sentinel; Backend is "mysql". +func errUnsupported(op string) error { + return &storage.ErrUnsupported{Op: op, Backend: Backend} +} diff --git a/internal/storage/mysql/unsupported_gen.go b/internal/storage/mysql/unsupported_gen.go new file mode 100644 index 000000000..823cc4b8e --- /dev/null +++ b/internal/storage/mysql/unsupported_gen.go @@ -0,0 +1,236 @@ +// Code generated by gen (mysql typed-unsupported shell); DO NOT EDIT. + +package mysql + +import ( + "context" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// unsupportedDoltStorage is the generated typed-unsupported shell for storage.DoltStorage. +// Every method returns *storage.ErrUnsupported; embed it and override the +// real slice. DO NOT hand-edit β€” regenerate with `go generate ./...`. +type unsupportedDoltStorage struct{} + +func (unsupportedDoltStorage) MergeSlotCreate(_ context.Context, _ string) (_ *types.Issue, err error) { + err = errUnsupported("MergeSlotCreate") + return +} + +func (unsupportedDoltStorage) MergeSlotCheck(_ context.Context) (_ *storage.MergeSlotStatus, err error) { + err = errUnsupported("MergeSlotCheck") + return +} + +func (unsupportedDoltStorage) MergeSlotAcquire(_ context.Context, _ string, _ string, _ bool) (_ *storage.MergeSlotResult, err error) { + err = errUnsupported("MergeSlotAcquire") + return +} + +func (unsupportedDoltStorage) MergeSlotRelease(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("MergeSlotRelease") + return +} + +func (unsupportedDoltStorage) Branch(_ context.Context, _ string) (err error) { + err = errUnsupported("Branch") + return +} + +func (unsupportedDoltStorage) Checkout(_ context.Context, _ string) (err error) { + err = errUnsupported("Checkout") + return +} + +func (unsupportedDoltStorage) CurrentBranch(_ context.Context) (_ string, err error) { + err = errUnsupported("CurrentBranch") + return +} + +func (unsupportedDoltStorage) DeleteBranch(_ context.Context, _ string) (err error) { + err = errUnsupported("DeleteBranch") + return +} + +func (unsupportedDoltStorage) ListBranches(_ context.Context) (_ []string, err error) { + err = errUnsupported("ListBranches") + return +} + +func (unsupportedDoltStorage) CommitExists(_ context.Context, _ string) (_ bool, err error) { + err = errUnsupported("CommitExists") + return +} + +func (unsupportedDoltStorage) GetCurrentCommit(_ context.Context) (_ string, err error) { + err = errUnsupported("GetCurrentCommit") + return +} + +func (unsupportedDoltStorage) Status(_ context.Context) (_ *storage.Status, err error) { + err = errUnsupported("Status") + return +} + +func (unsupportedDoltStorage) Log(_ context.Context, _ int) (_ []storage.CommitInfo, err error) { + err = errUnsupported("Log") + return +} + +func (unsupportedDoltStorage) Merge(_ context.Context, _ string) (_ []storage.Conflict, err error) { + err = errUnsupported("Merge") + return +} + +func (unsupportedDoltStorage) GetConflicts(_ context.Context) (_ []storage.Conflict, err error) { + err = errUnsupported("GetConflicts") + return +} + +func (unsupportedDoltStorage) ResolveConflicts(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("ResolveConflicts") + return +} + +func (unsupportedDoltStorage) History(_ context.Context, _ string) (_ []*storage.HistoryEntry, err error) { + err = errUnsupported("History") + return +} + +func (unsupportedDoltStorage) AsOf(_ context.Context, _ string, _ string) (_ *types.Issue, err error) { + err = errUnsupported("AsOf") + return +} + +func (unsupportedDoltStorage) Diff(_ context.Context, _ string, _ string) (_ []*storage.DiffEntry, err error) { + err = errUnsupported("Diff") + return +} + +func (unsupportedDoltStorage) AddRemote(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("AddRemote") + return +} + +func (unsupportedDoltStorage) RemoveRemote(_ context.Context, _ string) (err error) { + err = errUnsupported("RemoveRemote") + return +} + +func (unsupportedDoltStorage) HasRemote(_ context.Context, _ string) (_ bool, err error) { + err = errUnsupported("HasRemote") + return +} + +func (unsupportedDoltStorage) ListRemotes(_ context.Context) (_ []storage.RemoteInfo, err error) { + err = errUnsupported("ListRemotes") + return +} + +func (unsupportedDoltStorage) Push(_ context.Context) (err error) { + err = errUnsupported("Push") + return +} + +func (unsupportedDoltStorage) Pull(_ context.Context) (err error) { + err = errUnsupported("Pull") + return +} + +func (unsupportedDoltStorage) ForcePush(_ context.Context) (err error) { + err = errUnsupported("ForcePush") + return +} + +func (unsupportedDoltStorage) PushRemote(_ context.Context, _ string, _ bool) (err error) { + err = errUnsupported("PushRemote") + return +} + +func (unsupportedDoltStorage) PullRemote(_ context.Context, _ string) (err error) { + err = errUnsupported("PullRemote") + return +} + +func (unsupportedDoltStorage) Fetch(_ context.Context, _ string) (err error) { + err = errUnsupported("Fetch") + return +} + +func (unsupportedDoltStorage) PushTo(_ context.Context, _ string) (err error) { + err = errUnsupported("PushTo") + return +} + +func (unsupportedDoltStorage) PullFrom(_ context.Context, _ string) (_ []storage.Conflict, err error) { + err = errUnsupported("PullFrom") + return +} + +func (unsupportedDoltStorage) Sync(_ context.Context, _ string, _ string) (_ *storage.SyncResult, err error) { + err = errUnsupported("Sync") + return +} + +func (unsupportedDoltStorage) SyncStatus(_ context.Context, _ string) (_ *storage.SyncStatus, err error) { + err = errUnsupported("SyncStatus") + return +} + +func (unsupportedDoltStorage) AddFederationPeer(_ context.Context, _ *storage.FederationPeer) (err error) { + err = errUnsupported("AddFederationPeer") + return +} + +func (unsupportedDoltStorage) GetFederationPeer(_ context.Context, _ string) (_ *storage.FederationPeer, err error) { + err = errUnsupported("GetFederationPeer") + return +} + +func (unsupportedDoltStorage) ListFederationPeers(_ context.Context) (_ []*storage.FederationPeer, err error) { + err = errUnsupported("ListFederationPeers") + return +} + +func (unsupportedDoltStorage) RemoveFederationPeer(_ context.Context, _ string) (err error) { + err = errUnsupported("RemoveFederationPeer") + return +} + +func (unsupportedDoltStorage) CheckEligibility(_ context.Context, _ string, _ int) (_ bool, _ string, err error) { + err = errUnsupported("CheckEligibility") + return +} + +func (unsupportedDoltStorage) ApplyCompaction(_ context.Context, _ string, _ int, _ int, _ int, _ string) (err error) { + err = errUnsupported("ApplyCompaction") + return +} + +func (unsupportedDoltStorage) GetTier1Candidates(_ context.Context) (_ []*types.CompactionCandidate, err error) { + err = errUnsupported("GetTier1Candidates") + return +} + +func (unsupportedDoltStorage) GetTier2Candidates(_ context.Context) (_ []*types.CompactionCandidate, err error) { + err = errUnsupported("GetTier2Candidates") + return +} + +func (unsupportedDoltStorage) SnapshotIssue(_ context.Context, _ string, _ int) (err error) { + err = errUnsupported("SnapshotIssue") + return +} + +func (unsupportedDoltStorage) GetCompactionSnapshot(_ context.Context, _ string) (_ *types.IssueSnapshot, err error) { + err = errUnsupported("GetCompactionSnapshot") + return +} + +func (unsupportedDoltStorage) RestoreFromSnapshot(_ context.Context, _ string) (_ *types.IssueSnapshot, err error) { + err = errUnsupported("RestoreFromSnapshot") + return +} + +// NOTE: partial shell (104 methods skipped); the embedding composite must assert storage.DoltStorage itself. diff --git a/internal/storage/mysqldialect/credential.go b/internal/storage/mysqldialect/credential.go new file mode 100644 index 000000000..1a034233b --- /dev/null +++ b/internal/storage/mysqldialect/credential.go @@ -0,0 +1,83 @@ +package mysqldialect + +import ( + "fmt" + "net" + "strconv" + + "github.com/go-sql-driver/mysql" +) + +// HasPassword reports whether the DSN already carries a password. A DSN that already +// has one (a full BEADS_MYSQL_URL, or an explicit --mysql-url) wins outright: the +// credential ladder is not applied on top of it. +func HasPassword(dsn string) bool { + cfg, err := mysql.ParseDSN(dsn) + return err == nil && cfg.Passwd != "" +} + +// WithCredential returns dsn with the resolved credential placed: the password +// always, and the username only when non-empty (a Vault-style dynamic user/password +// pair) β€” an empty username leaves the DSN's user untouched. It uses go-sql-driver's +// own parser (never string surgery) and VERIFIES the round-trip. +// +// The grammar has one sharp edge pgdialect does not: it cannot carry a password +// without a username (FormatDSN emits the user:pass@ block only when a user is +// present), so a userless result is refused. But a provided username satisfies that +// requirement, making a bare tcp(host)/ base DSN plus a dynamic user/password pair a +// valid configuration. This is the counterpart to mysql.RedactPassword. +func WithCredential(dsn, username, password string) (string, error) { + cfg, err := mysql.ParseDSN(dsn) + if err != nil { + return "", fmt.Errorf("cannot place credential into the connection string (unparseable DSN): %w", err) + } + if username != "" { + cfg.User = username + } + if cfg.User == "" { + return "", fmt.Errorf("cannot place a password: the DSN has no username, and the MySQL DSN grammar cannot carry a password without one; add a user to the DSN") + } + cfg.Passwd = password + out := cfg.FormatDSN() + + again, err := mysql.ParseDSN(out) + if err != nil { + return "", fmt.Errorf("cannot place credential into the connection string (unparseable result): %w", err) + } + if again.Passwd != password { + return "", fmt.Errorf("password did not round-trip into the connection string; refusing to connect with a wrong or empty password") + } + if username != "" && again.User != username { + return "", fmt.Errorf("username did not round-trip into the connection string; refusing to connect as the wrong user") + } + return out, nil +} + +// WithPassword places only the password, preserving the DSN's user. It is +// WithCredential with no username override. +func WithPassword(dsn, password string) (string, error) { + return WithCredential(dsn, "", password) +} + +// HostPort returns the TCP endpoint go-sql-driver would dial for dsn, for keying the +// [host:port] credentials-file lookup. ok=false for non-TCP networks (a unix socket +// has no port) or an unparseable DSN; the caller degrades the file rung to +// not-configured rather than failing. +func HostPort(dsn string) (host string, port int, ok bool) { + cfg, err := mysql.ParseDSN(dsn) + if err != nil { + return "", 0, false + } + if cfg.Net != "tcp" { + return "", 0, false + } + h, p, err := net.SplitHostPort(cfg.Addr) + if err != nil { + return "", 0, false + } + port, err = strconv.Atoi(p) + if err != nil { + return "", 0, false + } + return h, port, true +} diff --git a/internal/storage/mysqldialect/credential_extra_test.go b/internal/storage/mysqldialect/credential_extra_test.go new file mode 100644 index 000000000..f43019e24 --- /dev/null +++ b/internal/storage/mysqldialect/credential_extra_test.go @@ -0,0 +1,81 @@ +package mysqldialect + +import ( + "testing" + + "github.com/go-sql-driver/mysql" +) + +func mustUser(t *testing.T, out string) string { + t.Helper() + cfg, err := mysql.ParseDSN(out) + if err != nil { + t.Fatalf("mysql.ParseDSN(%q): %v", out, err) + } + return cfg.User +} + +func TestWithCredentialOverridesUser(t *testing.T) { + out, err := WithCredential("old@tcp(h:3306)/db", "vu", "pw") + if err != nil { + t.Fatal(err) + } + if u := mustUser(t, out); u != "vu" { + t.Fatalf("user = %q, want vu", u) + } + if pw := mustPassword(t, out); pw != "pw" { + t.Fatalf("password = %q, want pw", pw) + } +} + +func TestWithCredentialEmptyUsernameKeepsUser(t *testing.T) { + out, err := WithCredential("old@tcp(h:3306)/db", "", "pw") + if err != nil { + t.Fatal(err) + } + if u := mustUser(t, out); u != "old" { + t.Fatalf("user = %q, want old (preserved)", u) + } +} + +// A provided username satisfies the grammar's user requirement, making a userless base +// DSN plus a dynamic user/password pair valid. +func TestWithCredentialUsernameSatisfiesUserlessDSN(t *testing.T) { + out, err := WithCredential("tcp(h:3306)/db", "vu", "pw") + if err != nil { + t.Fatal(err) + } + if u := mustUser(t, out); u != "vu" { + t.Fatalf("user = %q, want vu", u) + } + if pw := mustPassword(t, out); pw != "pw" { + t.Fatalf("password = %q, want pw", pw) + } +} + +// Password-only against a userless DSN still fails (the guard is intact). +func TestWithCredentialUserlessStillFails(t *testing.T) { + if _, err := WithCredential("tcp(h:3306)/db", "", "pw"); err == nil { + t.Fatal("expected error: a userless DSN cannot carry a password without a username") + } +} + +func TestHostPort(t *testing.T) { + cases := []struct { + dsn string + host string + port int + ok bool + }{ + {"u@tcp(10.0.0.5:55441)/", "10.0.0.5", 55441, true}, + {"u@/", "127.0.0.1", 3306, true}, // tcp Addr defaults to 127.0.0.1:3306 + {"u@tcp(dbhost)/", "dbhost", 3306, true}, // ensureHavePort appends :3306 + {"u@unix(/tmp/mysql.sock)/", "", 0, false}, // non-tcp -> not ok + } + for _, c := range cases { + h, p, ok := HostPort(c.dsn) + if ok != c.ok || (ok && (h != c.host || p != c.port)) { + t.Errorf("HostPort(%q) = (%q,%d,%v), want (%q,%d,%v)", c.dsn, h, p, ok, c.host, c.port, c.ok) + } + } +} diff --git a/internal/storage/mysqldialect/credential_test.go b/internal/storage/mysqldialect/credential_test.go new file mode 100644 index 000000000..49ef6affd --- /dev/null +++ b/internal/storage/mysqldialect/credential_test.go @@ -0,0 +1,82 @@ +package mysqldialect + +import ( + "testing" + + "github.com/go-sql-driver/mysql" +) + +func TestHasPassword(t *testing.T) { + cases := []struct { + dsn string + want bool + }{ + {"bts:secret@tcp(127.0.0.1:3306)/db", true}, + {"bts@tcp(127.0.0.1:3306)/db", false}, + {"bts@/", false}, // no-tcp form parses, no password + {"", false}, // empty parses to a default config + {"bts:pw@tcp(127.0.0.1:3306)", false}, // missing trailing slash -> parse error + } + for _, c := range cases { + if got := HasPassword(c.dsn); got != c.want { + t.Errorf("HasPassword(%q) = %v, want %v", c.dsn, got, c.want) + } + } +} + +// mustPassword parses out and returns the password go-sql-driver sees. +func mustPassword(t *testing.T, out string) string { + t.Helper() + cfg, err := mysql.ParseDSN(out) + if err != nil { + t.Fatalf("mysql.ParseDSN(%q): %v", out, err) + } + return cfg.Passwd +} + +func TestWithPasswordBasic(t *testing.T) { + out, err := WithPassword("bts@tcp(127.0.0.1:55441)/", "s3cr3t") + if err != nil { + t.Fatal(err) + } + if pw := mustPassword(t, out); pw != "s3cr3t" { + t.Fatalf("password = %q, want s3cr3t", pw) + } + cfg, _ := mysql.ParseDSN(out) + if cfg.User != "bts" { + t.Fatalf("user = %q, want bts (must be preserved)", cfg.User) + } +} + +// go-sql-driver writes the password verbatim; a token full of DSN-significant +// characters (as an RDS-IAM token carries) must round-trip byte-for-byte. +func TestWithPasswordSpecialChars(t *testing.T) { + for _, tok := range []string{"p@ss:w/rd?x=1&y=2", "X-Amz=abc&Sig=d/e+f%20", `a'b\c d`} { + out, err := WithPassword("bts@tcp(h:3306)/db", tok) + if err != nil { + t.Fatalf("token %q: %v", tok, err) + } + if pw := mustPassword(t, out); pw != tok { + t.Fatalf("password = %q, want %q", pw, tok) + } + } +} + +func TestWithPasswordReplacesExisting(t *testing.T) { + out, err := WithPassword("bts:old@tcp(h:3306)/db", "new") + if err != nil { + t.Fatal(err) + } + if pw := mustPassword(t, out); pw != "new" { + t.Fatalf("password = %q, want new", pw) + } +} + +// The MySQL sharp edge: a userless DSN cannot carry a password (FormatDSN drops it), +// so WithPassword must refuse rather than silently connect without one. +func TestWithPasswordUserlessDSNFails(t *testing.T) { + _, err := WithPassword("tcp(127.0.0.1:55441)/", "orphan") + if err == nil { + t.Fatal("expected an error for a userless DSN (password would be silently dropped)") + } +} diff --git a/internal/storage/mysqldialect/driver.go b/internal/storage/mysqldialect/driver.go new file mode 100644 index 000000000..2b9c286db --- /dev/null +++ b/internal/storage/mysqldialect/driver.go @@ -0,0 +1,91 @@ +// Package mysqldialect is the MySQL backend's thin driver wrapper. bd's canonical SQL +// is MySQL, so unlike pgdialect there is no translation β€” the ONE rewrite is the +// self-referential is_blocked recompute UPDATE, which MySQL 8 rejects (error 1093: a +// table may not be updated while referenced in its own subquery) but Dolt and Postgres +// permit. The rewrite is isolated here, so the shared issueops/sqlkit layers and the +// Dolt/Postgres backends are entirely untouched. +// +// A portable, perf-reviewed rewrite in shared issueops is tracked as a backlog item; +// this wrapper is the per-backend containment chosen to avoid touching Dolt's hot path. +package mysqldialect + +import ( + "context" + "database/sql" + "database/sql/driver" + + "github.com/go-sql-driver/mysql" +) + +// Open returns a *sql.DB over go-sql-driver whose Prepare rewrites the recompute +// UPDATE (see rewriteSelfRefUpdate); every other statement passes through verbatim. +// Like pgdialect, the wrapper exposes no Execer/Queryer fast path, so database/sql +// routes every statement through Prepare β€” the one place the rewrite happens. +func Open(dsn string) (*sql.DB, error) { + base, err := (mysql.MySQLDriver{}).OpenConnector(dsn) + if err != nil { + return nil, err + } + return sql.OpenDB(&connector{base: base}), nil +} + +type connector struct{ base driver.Connector } + +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + cn, err := c.base.Connect(ctx) + if err != nil { + return nil, err + } + return &conn{base: cn}, nil +} + +func (c *connector) Driver() driver.Driver { return c.base.Driver() } + +// conn wraps the mysql driver.Conn and rewrites the recompute UPDATE at prepare time. +// It deliberately implements no Execer/Queryer so database/sql prepares every stmt. +type conn struct{ base driver.Conn } + +func (c *conn) Prepare(query string) (driver.Stmt, error) { + return c.base.Prepare(rewriteSelfRefUpdate(query)) +} + +func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + q := rewriteSelfRefUpdate(query) + if p, ok := c.base.(driver.ConnPrepareContext); ok { + return p.PrepareContext(ctx, q) + } + return c.base.Prepare(q) +} + +func (c *conn) Close() error { return c.base.Close() } + +//nolint:staticcheck // SA1019: driver.Conn requires the deprecated Begin; BeginTx forwarded below. +func (c *conn) Begin() (driver.Tx, error) { return c.base.Begin() } + +func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if b, ok := c.base.(driver.ConnBeginTx); ok { + return b.BeginTx(ctx, opts) + } + return c.base.Begin() //nolint:staticcheck +} + +func (c *conn) ResetSession(ctx context.Context) error { + if rs, ok := c.base.(driver.SessionResetter); ok { + return rs.ResetSession(ctx) + } + return nil +} + +func (c *conn) IsValid() bool { + if v, ok := c.base.(driver.Validator); ok { + return v.IsValid() + } + return true +} + +func (c *conn) Ping(ctx context.Context) error { + if p, ok := c.base.(driver.Pinger); ok { + return p.Ping(ctx) + } + return nil +} diff --git a/internal/storage/mysqldialect/rewrite.go b/internal/storage/mysqldialect/rewrite.go new file mode 100644 index 000000000..500bced7c --- /dev/null +++ b/internal/storage/mysqldialect/rewrite.go @@ -0,0 +1,43 @@ +package mysqldialect + +import ( + "fmt" + "regexp" + "strings" +) + +// selfRefUpdateRe matches bd's is_blocked recompute shape: +// +// UPDATE
SET <...> WHERE .id IN () +// +// The SET is non-greedy up to the first WHERE (SET clauses contain no WHERE); the +// placeholder list is the batched `?,?,…`. +var selfRefUpdateRe = regexp.MustCompile(`(?is)^(\s*UPDATE\s+(\w+)\s+(\w+)\s+SET\s+.+?)\s+WHERE\s+(\w+)\.id\s+IN\s+\(([^)]*)\)\s*(.*)$`) + +// rewriteSelfRefUpdate rewrites the is_blocked recompute UPDATE into a MySQL-8-safe +// double-nested form so the target table's self-reference is materialized (avoiding +// error 1093). Everything else is returned verbatim. The transform is semantics- +// preserving: the id set (batch ∩ conditions) is computed in a materialized derived +// table, and the outer UPDATE writes exactly those rows. Positional placeholders keep +// their order because the WHERE body is moved intact. +func rewriteSelfRefUpdate(q string) string { + // Fast reject: only the is_blocked recompute both sets is_blocked and self-joins. + if !strings.Contains(q, "is_blocked") { + return q + } + m := selfRefUpdateRe.FindStringSubmatch(q) + if m == nil { + return q + } + head, table, alias, whereAlias, placeholders, rest := m[1], m[2], m[3], m[4], m[5], m[6] + // Only the recompute has further conditions after the IN clause (the self-ref + // EXISTS block); a plain `UPDATE … WHERE id IN (…)` needs no rewrite. + if whereAlias != alias || strings.TrimSpace(rest) == "" { + return q + } + // Rebind the outer alias to the derived-table alias inside the moved conditions. + reb := regexp.MustCompile(`\b` + regexp.QuoteMeta(alias) + `\.`) + restRebound := reb.ReplaceAllString(rest, "_b.") + return fmt.Sprintf("%s WHERE %s.id IN (SELECT id FROM (SELECT _b.id FROM %s _b WHERE _b.id IN (%s) %s) AS _blk)", + head, alias, table, placeholders, restRebound) +} diff --git a/internal/storage/mysqldialect/rewrite_test.go b/internal/storage/mysqldialect/rewrite_test.go new file mode 100644 index 000000000..3e6357f29 --- /dev/null +++ b/internal/storage/mysqldialect/rewrite_test.go @@ -0,0 +1,123 @@ +package mysqldialect + +import ( + "regexp" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" +) + +func TestRewriteSelfRefUpdate(t *testing.T) { + recompute := `UPDATE issues i SET i.is_blocked = 1, i.updated_at = i.updated_at + WHERE i.id IN (?,?,?) + AND i.is_blocked = 0 + AND i.status <> 'closed' AND i.status <> 'pinned' + AND (EXISTS (SELECT 1 FROM dependencies d JOIN issues t ON t.id = d.depends_on_issue_id + WHERE d.issue_id = i.id AND d.type = 'blocks' AND t.status <> 'closed'))` + + got := rewriteSelfRefUpdate(recompute) + if got == recompute { + t.Fatal("recompute UPDATE was not rewritten") + } + // The outer target must no longer be referenced directly in a subquery: the + // materialized derived table carries the self-reference. + if !strings.Contains(got, "SELECT id FROM (SELECT _b.id FROM issues _b") { + t.Fatalf("missing double-nested derived table:\n%s", got) + } + // The correlation must be rebound to the derived-table alias. + if !strings.Contains(got, "d.issue_id = _b.id") { + t.Fatalf("alias not rebound to _b:\n%s", got) + } + // The outer SET still targets the real alias. + if !strings.HasPrefix(strings.TrimSpace(got), "UPDATE issues i SET i.is_blocked = 1") { + t.Fatalf("outer UPDATE target changed:\n%s", got) + } + // Placeholder count preserved (positional binding). + if a, b := strings.Count(recompute, "?"), strings.Count(got, "?"); a != b { + t.Fatalf("placeholder count changed: %d -> %d", a, b) + } +} + +// TestRewriteMatchesRealRecomputeTemplates drives the rewrite from the ACTUAL +// issueops is_blocked recompute templates, not a hand-copied fixture. The rewrite +// regex is coupled to the exact shape those templates emit (UPDATE SET +// … WHERE .id IN (…) ); if a template later grows a predicate +// before `id IN`, backticks the table, renames the alias, or changes whitespace, the +// regex would silently stop matching and MySQL 8 would raise error 1093 at runtime on +// a core write path. That failure would otherwise only surface under the env-gated +// MySQL conformance suite β€” this non-gated test makes template drift fail `go test`. +func TestRewriteMatchesRealRecomputeTemplates(t *testing.T) { + templates := issueops.SelfReferentialRecomputeTemplates() + if len(templates) == 0 { + t.Fatal("issueops.SelfReferentialRecomputeTemplates() returned no templates") + } + // Parse the UPDATE target table+alias so the assertions can target the OUTER + // self-reference specifically (the inner materialized `_b.id IN (?…)` is expected). + updateHead := regexp.MustCompile(`(?is)^\s*UPDATE\s+(\w+)\s+(\w+)\s+SET\b`) + for i, tmpl := range templates { + m := updateHead.FindStringSubmatch(tmpl) + if m == nil { + t.Errorf("template[%d] is not an `UPDATE
SET …` shape:\n%s", i, tmpl) + continue + } + table, alias := m[1], m[2] + + got := rewriteSelfRefUpdate(tmpl) + if got == tmpl { + t.Errorf("template[%d] was NOT rewritten β€” the rewrite regex no longer matches an is_blocked "+ + "recompute template; MySQL error 1093 would return at runtime:\n%s", i, tmpl) + continue + } + // The target table must be materialized in a derived table so the outer UPDATE + // no longer references it directly in a subquery (the 1093 avoidance). + if want := "SELECT id FROM (SELECT _b.id FROM " + table + " _b"; !strings.Contains(got, want) { + t.Errorf("template[%d] rewrite missing the double-nested derived table %q:\n%s", i, want, got) + } + // The OUTER target alias must now select from the materialized set, not a bare + // self-referential `.id IN (?…)`. + outerMaterialized := regexp.MustCompile(`(?i)\b` + alias + `\.id\s+IN\s+\(\s*SELECT`) + if !outerMaterialized.MatchString(got) { + t.Errorf("template[%d] outer %s.id IN (…) was not materialized into a subselect:\n%s", i, alias, got) + } + outerRaw := regexp.MustCompile(`(?i)\b` + alias + `\.id\s+IN\s+\(\s*\?`) + if outerRaw.MatchString(got) { + t.Errorf("template[%d] outer %s.id IN (?…) is still a raw self-reference after rewrite (MySQL 1093):\n%s", i, alias, got) + } + // Positional binding is preserved (placeholders unchanged). + if a, b := strings.Count(tmpl, "?"), strings.Count(got, "?"); a != b { + t.Errorf("template[%d] placeholder count changed: %d -> %d", i, a, b) + } + } +} + +func TestRewritePassThrough(t *testing.T) { + cases := []string{ + "SELECT * FROM issues WHERE id = ?", + "UPDATE issues SET title = ? WHERE id = ?", // no is_blocked, no IN + "UPDATE issues i SET i.is_blocked = 1 WHERE i.id IN (?,?)", // batch update, no self-ref conditions + "INSERT INTO issues (id) VALUES (?)", + "DELETE FROM issues WHERE id = ?", + } + for _, q := range cases { + if got := rewriteSelfRefUpdate(q); got != q { + t.Errorf("query was rewritten but should pass through:\n in: %s\nout: %s", q, got) + } + } +} + +// The rewrite must not leave the outer target table referenced in a subquery in a way +// that regex could double-apply; a second pass is a no-op on already-rewritten SQL. +func TestRewriteIdempotentShape(t *testing.T) { + q := `UPDATE wisps w SET w.is_blocked = 0, w.updated_at = w.updated_at + WHERE w.id IN (?) AND w.is_blocked = 1 AND (w.status = 'closed')` + once := rewriteSelfRefUpdate(q) + if once == q { + t.Fatal("expected rewrite") + } + // A plain "?,?" derived form should still parse-shape sanely (no target alias left + // as a bare self-join outside the derived table). + if regexp.MustCompile(`WHERE w\.id IN \(\?`).MatchString(once) { + t.Fatalf("outer WHERE still has a raw placeholder IN (self-ref not materialized):\n%s", once) + } +} diff --git a/internal/storage/pgdialect/corpus_test.go b/internal/storage/pgdialect/corpus_test.go new file mode 100644 index 000000000..f3acb1c6d --- /dev/null +++ b/internal/storage/pgdialect/corpus_test.go @@ -0,0 +1,426 @@ +package pgdialect_test + +// This file is the PREPARE gate the package doc promises. It enumerates every +// distinct statement family bd's shared layer (sqlbuild + issueops) emits on +// the read/search path, runs each through pgdialect.Translate, and β€” when a +// live Postgres is configured via BEADS_PG_TEST_URL β€” PREPAREs the translated +// text against a scratch schema. An untranslatable construct fails the test +// loudly (Translate error, surviving MySQL residue, or a PREPARE-time Postgres +// error) instead of silently mistranslating a row set. +// +// It is an external test package (pgdialect_test) so it can live alongside the +// white-box translate_test.go while only touching pgdialect's exported surface. +// +// The scratch schema is built here rather than via postgres.InitSchema on +// purpose: pgdialect is a lower layer than the postgres backend (postgres +// imports pgdialect), so importing postgres into pgdialect's tests would invert +// the dependency and couple this gate to that package's build state. The tables +// are generated from sqlbuild.IssueSelectColumns so they cannot drift out of +// sync with the column list the queries project. Only the shapes the corpus +// exercises are created; types are faithful where an operator depends on them +// (jsonb metadata, integer priority, smallint flags, timestamp columns) and +// text elsewhere. + +import ( + "context" + "database/sql" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/pgdialect" + "github.com/steveyegge/beads/internal/storage/sqlbuild" + "github.com/steveyegge/beads/internal/types" +) + +type corpusStmt struct { + name string + sql string +} + +func strp(s string) *string { return &s } +func intp(i int) *int { return &i } + +// searchSelectSQL mirrors issueops.searchTableInTx Pattern A: the full-column +// filtered scan for one table family (incl. the label-driven JOIN plan). +func searchSelectSQL(tables sqlbuild.FilterTables, query string, filter types.IssueFilter) (string, error) { + plan := sqlbuild.BuildLabelDrivenSearch(filter, tables) + where, _, err := sqlbuild.BuildIssueFilterClauses(query, plan.Filter, tables) + if err != nil { + return "", err + } + where, _ = plan.MergeInto(where, nil) + whereSQL := "" + if len(where) > 0 { + whereSQL = "WHERE " + strings.Join(where, " AND ") + } + selectKw := "SELECT " + if plan.Distinct { + selectKw = "SELECT DISTINCT " + } + return fmt.Sprintf("%s%s FROM %s %s %s", + selectKw, sqlbuild.IssueSelectColumns, plan.FromSQL, whereSQL, + sqlbuild.OrderBy(filter.SortBy, filter.SortDesc, "")), nil +} + +// searchCountsSQL mirrors issueops.runFilterSearchQueryInTx: the counts +// mega-query (the DATE_FORMAT / JSON_OBJECT / CAST / JSON_ARRAYAGG surface). +func searchCountsSQL(tables sqlbuild.FilterTables, query string, filter types.IssueFilter, includeWispReverseDeps bool) (string, error) { + where, _, err := sqlbuild.BuildIssueFilterClauses(query, filter, tables) + if err != nil { + return "", err + } + whereSQL := "" + if len(where) > 0 { + whereSQL = "WHERE " + strings.Join(where, " AND ") + } + sql, _ := sqlbuild.SearchCountsSQL(tables, nil, whereSQL, sqlbuild.OrderBy(filter.SortBy, filter.SortDesc, "i"), "", includeWispReverseDeps, filter.SkipLabels) + return sql, nil +} + +// readySelectSQL mirrors issueops.GetReadyWorkInTx: SELECT id FROM
with +// the ready-work WHERE/ORDER. +func readySelectSQL(tables sqlbuild.FilterTables, filter types.WorkFilter) (string, error) { + whereSQL, _, err := sqlbuild.BuildReadyWorkWhere(filter, tables, sqlbuild.ReadyWorkWhereInputs{}) + if err != nil { + return "", err + } + order := sqlbuild.BuildReadyWorkOrder(filter.SortPolicy, "created_at", "priority") + return fmt.Sprintf("SELECT id FROM %s %s %s", tables.Main, whereSQL, order.SQL), nil +} + +// readyCountsSQL mirrors issueops.GetReadyWorkWithCountsInTx: the counts +// mega-query fed the ready-work predicates. +func readyCountsSQL(tables sqlbuild.FilterTables, filter types.WorkFilter, includeWispReverseDeps bool) (string, error) { + whereSQL, _, err := sqlbuild.BuildReadyWorkWhere(filter, tables, sqlbuild.ReadyWorkWhereInputs{}) + if err != nil { + return "", err + } + order := sqlbuild.BuildReadyWorkOrder(filter.SortPolicy, "created_at", "priority") + sql, _ := sqlbuild.SearchCountsSQL(tables, nil, whereSQL, order.SQL, "", includeWispReverseDeps, false) + return sql, nil +} + +// adaptiveIDInstrSQL is issueops.GetAdaptiveIDLengthTx's create-path query +// (CONCAT + INSTR + SUBSTRING). Reproduced verbatim so the corpus covers the +// non-search statement family the smoke test's explicit-ID creates never hit. +const adaptiveIDInstrSQL = `SELECT COUNT(*) + FROM issues + WHERE id LIKE CONCAT(?, '-%') + AND INSTR(SUBSTRING(id, LENGTH(?) + 2), '.') = 0` + +func buildCorpus(t *testing.T) []corpusStmt { + t.Helper() + + // Representative filters: priority, an AND label, metadata has-key + eq, + // and a parent filter (each drives a distinct clause shape). + richIssue := types.IssueFilter{ + Priority: intp(1), + Labels: []string{"backlog"}, + HasMetadataKey: "sprint", + MetadataFields: map[string]string{"team": "core"}, + ParentID: strp("bd-1"), + SortBy: "priority", + } + richWork := types.WorkFilter{ + Status: types.Status("open"), + Priority: intp(1), + Labels: []string{"backlog"}, + HasMetadataKey: "sprint", + MetadataFields: map[string]string{"team": "core"}, + ParentID: strp("bd-1"), + } + metaOnly := types.IssueFilter{HasMetadataKey: "sprint", MetadataFields: map[string]string{"team": "core"}} + + var corpus []corpusStmt + add := func(name, sqlText string, err error) { + if err != nil { + t.Fatalf("build corpus %q: %v", name, err) + } + corpus = append(corpus, corpusStmt{name: name, sql: sqlText}) + } + + s, err := searchSelectSQL(sqlbuild.IssuesFilterTables, "", types.IssueFilter{}) + add("search/issues/plain", s, err) + s, err = searchSelectSQL(sqlbuild.IssuesFilterTables, "bd-1", richIssue) + add("search/issues/rich", s, err) + s, err = searchSelectSQL(sqlbuild.WispsFilterTables, "", richIssue) + add("search/wisps/rich", s, err) + s, err = searchSelectSQL(sqlbuild.IssuesFilterTables, "", metaOnly) + add("search/issues/metadata-only", s, err) + + s, err = searchCountsSQL(sqlbuild.IssuesFilterTables, "", types.IssueFilter{}, false) + add("counts/issues/plain", s, err) + s, err = searchCountsSQL(sqlbuild.IssuesFilterTables, "", richIssue, true) + add("counts/issues/rich", s, err) + s, err = searchCountsSQL(sqlbuild.WispsFilterTables, "", richIssue, true) + add("counts/wisps/rich", s, err) + + s, err = readySelectSQL(sqlbuild.IssuesFilterTables, richWork) + add("ready/issues", s, err) + s, err = readySelectSQL(sqlbuild.WispsFilterTables, richWork) + add("ready/wisps", s, err) + s, err = readyCountsSQL(sqlbuild.IssuesFilterTables, richWork, true) + add("ready-counts/issues", s, err) + s, err = readyCountsSQL(sqlbuild.WispsFilterTables, richWork, true) + add("ready-counts/wisps", s, err) + + corpus = append(corpus, + corpusStmt{"adaptive-id/instr", adaptiveIDInstrSQL}, + // Literal-path gate check (issueops/blocked_state.go): keeps the literal + // JSON rule covered alongside the dynamic one. + corpusStmt{"metadata/literal-gate", `SELECT id FROM issues WHERE JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.gate')) = ?`}, + // Dep JSON aggregation in isolation (DATE_FORMAT + JSON_OBJECT + CAST). + corpusStmt{"dep-json/arrayagg", fmt.Sprintf(`SELECT issue_id, JSON_ARRAYAGG(%s) AS deps_json FROM dependencies GROUP BY issue_id`, sqlbuild.DepJSONObject)}, + ) + return corpus +} + +// forbiddenResidue are lowercased markers of MySQL constructs that must never +// survive translation. JSON_ARRAYAGG is intentionally absent: Postgres 16 +// supports SQL-standard JSON_ARRAYAGG natively, so the wedge leaves it in place. +var forbiddenResidue = []string{ + "date_format(", "json_object(", "json_unquote(", "json_extract(", + " as char", "cast(", "insert ignore", "replace into", "on duplicate", + "utc_timestamp", "now()", "`", "?", +} + +func assertNoResidue(t *testing.T, name, translated string) { + t.Helper() + low := strings.ToLower(translated) + for _, marker := range forbiddenResidue { + if strings.Contains(low, marker) { + t.Errorf("%s: untranslated MySQL residue %q survived:\n%s", name, strings.TrimSpace(marker), translated) + } + } +} + +// TestCorpusTranslates runs with no database: it proves every statement family +// translates without error and leaves no MySQL residue. The fail-loud property +// is a build-time guarantee, so this half of the gate always runs. +func TestCorpusTranslates(t *testing.T) { + for _, c := range buildCorpus(t) { + translated, err := pgdialect.Translate(c.sql) + if err != nil { + t.Errorf("%s: Translate error: %v\n-- sql:\n%s", c.name, err, c.sql) + continue + } + assertNoResidue(t, c.name, translated) + } +} + +// TestCorpusPrepare is the live half: PREPARE every translated statement against +// a real Postgres so a construct that is syntactically valid but references a +// missing column/function/operator still fails loudly. Gated on BEADS_PG_TEST_URL. +func TestCorpusPrepare(t *testing.T) { + dsn := os.Getenv("BEADS_PG_TEST_URL") + if dsn == "" { + t.Skip("BEADS_PG_TEST_URL not set; skipping live Postgres PREPARE gate") + } + ctx := context.Background() + schema := fmt.Sprintf("corpus_%d", time.Now().UnixNano()) + + db, err := pgdialect.Open(dsn, schema) + if err != nil { + t.Fatalf("pgdialect.Open: %v", err) + } + t.Cleanup(func() { + _, _ = db.ExecContext(context.Background(), fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) + _ = db.Close() + }) + if err := createScratchSchema(ctx, db, schema); err != nil { + t.Fatalf("create scratch schema: %v", err) + } + + for _, c := range buildCorpus(t) { + t.Run(c.name, func(t *testing.T) { + translated, err := pgdialect.Translate(c.sql) + if err != nil { + t.Fatalf("Translate error: %v\n-- sql:\n%s", err, c.sql) + } + assertNoResidue(t, c.name, translated) + // PREPARE the RAW statement through the translating driver β€” the exact + // production path (driver.Translate then server Parse). + stmt, err := db.PrepareContext(ctx, c.sql) + if err != nil { + t.Fatalf("PREPARE failed: %v\n-- translated:\n%s", err, translated) + } + _ = stmt.Close() + }) + } +} + +// createScratchSchema builds just enough of the wedge schema for the corpus to +// PREPARE against: the issues/wisps families (generated from IssueSelectColumns +// so they track the projected column list), their label/dependency/comment +// side tables, and the bd_mysql_jsonkey helper the dynamic-path rewrite calls +// (in production schema.go/InitSchema owns this function). +func createScratchSchema(ctx context.Context, db *sql.DB, schema string) error { + stmts := []string{ + fmt.Sprintf(`CREATE SCHEMA IF NOT EXISTS %q`, schema), + issueTableDDL("issues"), + issueTableDDL("wisps"), + `CREATE TABLE labels (issue_id text, label text)`, + `CREATE TABLE wisp_labels (issue_id text, label text)`, + depTableDDL("dependencies"), + depTableDDL("wisp_dependencies"), + `CREATE TABLE comments (issue_id text)`, + `CREATE TABLE wisp_comments (issue_id text)`, + `CREATE OR REPLACE FUNCTION bd_mysql_jsonkey(p text) RETURNS text LANGUAGE sql IMMUTABLE AS $$ + SELECT CASE WHEN p LIKE '$."%"' THEN substring(p from 4 for length(p)-4) ELSE substring(p from 3) END + $$`, + } + for _, s := range stmts { + if _, err := db.ExecContext(ctx, s); err != nil { + return fmt.Errorf("exec %q: %w", firstLine(s), err) + } + } + return nil +} + +// issueTableDDL renders a CREATE TABLE for the issues/wisps family from +// IssueSelectColumns plus the derived is_blocked column the ready-work WHERE +// reads. Column types are faithful where an operator depends on them. +func issueTableDDL(table string) string { + var defs []string + for _, name := range splitColumnList(sqlbuild.IssueSelectColumns) { + defs = append(defs, name+" "+pgColumnType(name)) + } + defs = append(defs, "is_blocked smallint NOT NULL DEFAULT 0") + return fmt.Sprintf("CREATE TABLE %s (%s)", table, strings.Join(defs, ", ")) +} + +func depTableDDL(table string) string { + return fmt.Sprintf(`CREATE TABLE %s ( + issue_id text, depends_on_issue_id text, depends_on_wisp_id text, + depends_on_external text, type text, created_by text, + created_at timestamp(0), thread_id text, metadata jsonb)`, table) +} + +func pgColumnType(name string) string { + switch name { + case "metadata": + return "jsonb" + case "created_at", "updated_at", "started_at", "closed_at", "compacted_at", "due_at", "defer_until": + return "timestamp(0)" + case "priority", "estimated_minutes", "compaction_level", "original_size", "timeout_ns": + return "integer" + case "pinned", "ephemeral", "is_template", "no_history": + return "smallint" + default: + return "text" + } +} + +// splitColumnList splits a select-column constant into bare column names. +func splitColumnList(cols string) []string { + raw := strings.NewReplacer("\n", " ", "\t", " ").Replace(cols) + var names []string + for _, p := range strings.Split(raw, ",") { + if n := strings.TrimSpace(p); n != "" { + names = append(names, n) + } + } + return names +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} + +// TestTranslateNewRules pins the four red-team translator rules (DATE_FORMAT, +// JSON_OBJECT comma form, CAST AS CHAR, dynamic ?-path JSON metadata), including +// their fail-loud (left-untranslated) branches. +func TestTranslateNewRules(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + { + name: "DATE_FORMAT ISO8601 -> to_char", + in: `SELECT DATE_FORMAT(created_at, '%Y-%m-%dT%H:%i:%sZ') FROM issues`, + want: `SELECT to_char(created_at, 'YYYY-MM-DD"T"HH24:MI:SS"Z"') FROM issues`, + }, + { + name: "DATE_FORMAT unknown format left loud", + in: `SELECT DATE_FORMAT(created_at, '%Y-%m-%d') FROM issues`, + want: `SELECT DATE_FORMAT(created_at, '%Y-%m-%d') FROM issues`, + }, + { + name: "JSON_OBJECT even -> jsonb_build_object", + in: `SELECT JSON_OBJECT('a', x, 'b', y) FROM t`, + want: `SELECT jsonb_build_object('a', x, 'b', y) FROM t`, + }, + { + name: "JSON_OBJECT odd arg count left loud", + in: `SELECT JSON_OBJECT('a', x, 'b') FROM t`, + want: `SELECT JSON_OBJECT('a', x, 'b') FROM t`, + }, + { + name: "CAST AS CHAR -> ::text", + in: `SELECT CAST(metadata AS CHAR) FROM t`, + want: `SELECT (metadata)::text FROM t`, + }, + { + name: "CAST AS CHAR(n) -> ::text", + in: `SELECT CAST(metadata AS CHAR(64)) FROM t`, + want: `SELECT (metadata)::text FROM t`, + }, + { + name: "CAST AS SIGNED left loud", + in: `SELECT CAST(x AS SIGNED) FROM t`, + want: `SELECT CAST(x AS SIGNED) FROM t`, + }, + { + name: "dynamic has-key JSON_EXTRACT -> -> bd_mysql_jsonkey", + in: `SELECT id FROM issues WHERE JSON_EXTRACT(metadata, ?) IS NOT NULL`, + want: `SELECT id FROM issues WHERE (metadata -> bd_mysql_jsonkey($1)) IS NOT NULL`, + }, + { + name: "dynamic eq JSON_UNQUOTE(JSON_EXTRACT) -> ->> bd_mysql_jsonkey", + in: `SELECT id FROM issues WHERE JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) = ?`, + want: `SELECT id FROM issues WHERE (metadata ->> bd_mysql_jsonkey($1)) = $2`, + }, + { + name: "literal path rule still applies (regression)", + in: `SELECT 1 WHERE JSON_UNQUOTE(JSON_EXTRACT(d.metadata, '$.gate')) = 'any-children'`, + want: `SELECT 1 WHERE (d.metadata #>> '{gate}') = 'any-children'`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := pgdialect.Translate(tc.in) + if err != nil { + t.Fatalf("Translate error: %v", err) + } + if got != tc.want { + t.Errorf("mismatch\n in: %s\n got: %s\n want: %s", tc.in, got, tc.want) + } + }) + } +} + +// TestTranslateDepJSONObject exercises the real counts.go DepJSONObject constant +// (DATE_FORMAT + JSON_OBJECT + CAST composed) end to end. +func TestTranslateDepJSONObject(t *testing.T) { + got, err := pgdialect.Translate(sqlbuild.DepJSONObject) + if err != nil { + t.Fatalf("Translate error: %v", err) + } + for _, want := range []string{ + "jsonb_build_object(", + `to_char(created_at, 'YYYY-MM-DD"T"HH24:MI:SS"Z"')`, + "(metadata)::text", + } { + if !strings.Contains(got, want) { + t.Errorf("DepJSONObject translation missing %q:\n%s", want, got) + } + } + assertNoResidue(t, "DepJSONObject", got) +} diff --git a/internal/storage/pgdialect/credential.go b/internal/storage/pgdialect/credential.go new file mode 100644 index 000000000..d3b04c062 --- /dev/null +++ b/internal/storage/pgdialect/credential.go @@ -0,0 +1,121 @@ +package pgdialect + +import ( + "fmt" + "net/url" + "regexp" + "strings" + + "github.com/jackc/pgx/v5" +) + +// kvUserRe matches a libpq keyword/value user token, whose value is either +// single-quoted or a run of non-space chars. Like kvPasswordRe (redact.go) it is a +// token-level regex, not a full parser, so a ` user=` sequence inside another token's +// quoted value could be matched β€” an exotic case (a keyword-form DSN whose quoted +// value contains ` user=`, combined with a username-bearing credential). The +// round-trip verify in WithCredential catches the loud failures; the silent case is a +// documented sharp edge shared with kvPasswordRe. +var kvUserRe = regexp.MustCompile(`(?i)(^|\s)user\s*=\s*(?:'(?:[^'\\]|\\.)*'|\S*)`) + +// kvHasPasswordRe detects a libpq keyword/value connection password token (password=, +// not sslpassword=) for the DSN-embedded check in HasPassword. +var kvHasPasswordRe = regexp.MustCompile(`(?i)(^|\s)password\s*=`) + +// HasPassword reports whether the connection string itself EMBEDS a password β€” URL +// userinfo, a URL ?password= query param, or a libpq password= token. It deliberately +// does NOT report whether pgx would resolve a password from the ambient environment: +// pgx.ParseConfig folds PGPASSWORD and ~/.pgpass into cfg.Password, so parsing here +// would let a stale PGPASSWORD short-circuit the whole ladder and mask a configured +// command or credentials file. An embedded password is the only thing that wins +// outright; ambient PGPASSWORD/~/.pgpass remain the driver-native fallback, consulted +// at connect only when no ladder rung is configured. +func HasPassword(dsn string) bool { + if u, err := url.Parse(dsn); err == nil && u.Scheme != "" { + if u.User != nil { + if _, ok := u.User.Password(); ok { + return true + } + } + return u.Query().Get("password") != "" + } + return kvHasPasswordRe.MatchString(dsn) +} + +// WithCredential returns dsn with the resolved credential placed: the password +// always, and the username only when non-empty (a Vault-style dynamic user/password +// pair) β€” an empty username leaves the DSN's user untouched. It handles both grammars +// pgx accepts (URL userinfo for scheme:// DSNs, single-quoted keyword/value tokens +// otherwise) and VERIFIES with the same parser pgx uses that both fields round-tripped, +// failing loudly rather than emit a DSN that silently drops or mangles the secret. +func WithCredential(dsn, username, password string) (string, error) { + var out string + if u, err := url.Parse(dsn); err == nil && u.Scheme != "" { + // URL form: set the userinfo. url.String() percent-encodes special characters + // (an RDS-IAM token's '&'/'='/'/'), which pgx.ParseConfig decodes back β€” the + // verify below proves it. + user := username + if user == "" && u.User != nil { + user = u.User.Username() + } + u.User = url.UserPassword(user, password) + out = u.String() + } else { + // libpq keyword/value form: drop any existing password token (and, when we are + // overriding the user, the user token too), then append single-quoted tokens. + stripped := kvPasswordRe.ReplaceAllString(dsn, "$1") + if username != "" { + stripped = kvUserRe.ReplaceAllString(stripped, "$1") + } + stripped = strings.TrimSpace(strings.Join(strings.Fields(stripped), " ")) + if username != "" { + stripped += " user='" + escapeKeywordValue(username) + "'" + } + out = stripped + " password='" + escapeKeywordValue(password) + "'" + } + + cfg, err := pgx.ParseConfig(out) + if err != nil { + return "", fmt.Errorf("cannot place credential into the connection string (unparseable result): %w", err) + } + if cfg.Password != password { + return "", fmt.Errorf("password did not round-trip into the connection string; refusing to connect with a wrong or empty password") + } + if username != "" && cfg.User != username { + return "", fmt.Errorf("username did not round-trip into the connection string; refusing to connect as the wrong user") + } + return out, nil +} + +// WithPassword places only the password, preserving the DSN's user. It is +// WithCredential with no username override. This is the counterpart to +// RedactPassword: RedactPassword strips the password for persistence, WithPassword +// re-injects the resolved password at open time. +func WithPassword(dsn, password string) (string, error) { + return WithCredential(dsn, "", password) +} + +// HostPort returns the TCP endpoint pgx would connect to for dsn, for keying the +// [host:port] credentials-file lookup. ok=false when the DSN does not parse or +// resolves to a unix-socket directory (an absolute-path host) β€” a peer-auth socket +// has no place in a password file, and the caller degrades the file rung to +// not-configured. For a multi-host DSN this is the primary endpoint only. IPv6 hosts +// are returned bare (e.g. "::1"); the credentials-file key format does not bracket +// them, so an IPv6 endpoint effectively degrades to not-found in the file. +func HostPort(dsn string) (host string, port int, ok bool) { + cfg, err := pgx.ParseConfig(dsn) + if err != nil { + return "", 0, false + } + if cfg.Host == "" || strings.HasPrefix(cfg.Host, "/") { + return "", 0, false + } + return cfg.Host, int(cfg.Port), true +} + +// escapeKeywordValue escapes a value for a single-quoted libpq keyword/value token. +func escapeKeywordValue(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `'`, `\'`) + return s +} diff --git a/internal/storage/pgdialect/credential_extra_test.go b/internal/storage/pgdialect/credential_extra_test.go new file mode 100644 index 000000000..20251cd12 --- /dev/null +++ b/internal/storage/pgdialect/credential_extra_test.go @@ -0,0 +1,148 @@ +package pgdialect + +import ( + "os" + "path/filepath" + "testing" + + "github.com/jackc/pgx/v5" +) + +// TestMain neutralizes ambient PGPASSWORD/~/.pgpass so redaction and HasPassword tests +// are hermetic (pgx.ParseConfig folds them into a connection). The ambient-immunity +// tests re-set PGPASSWORD locally via t.Setenv. +func TestMain(m *testing.M) { + dir, err := os.MkdirTemp("", "beads-pgdialect-test") + if err != nil { + panic(err) + } + os.Unsetenv("PGPASSWORD") + os.Setenv("PGPASSFILE", filepath.Join(dir, "no-pgpass")) + code := m.Run() + _ = os.RemoveAll(dir) + os.Exit(code) +} + +func mustUser(t *testing.T, out string) string { + t.Helper() + cfg, err := pgx.ParseConfig(out) + if err != nil { + t.Fatalf("pgx.ParseConfig(%q): %v", out, err) + } + return cfg.User +} + +func TestWithCredentialURLForm(t *testing.T) { + out, err := WithCredential("postgres://old@h:5432/db", "vu", "pw") + if err != nil { + t.Fatal(err) + } + if u := mustUser(t, out); u != "vu" { + t.Fatalf("user = %q, want vu", u) + } + if pw := mustPassword(t, out); pw != "pw" { + t.Fatalf("password = %q, want pw", pw) + } +} + +func TestWithCredentialURLEmptyUsernameKeepsUser(t *testing.T) { + out, err := WithCredential("postgres://old@h:5432/db", "", "pw") + if err != nil { + t.Fatal(err) + } + if u := mustUser(t, out); u != "old" { + t.Fatalf("user = %q, want old (preserved)", u) + } +} + +func TestWithCredentialKeywordForm(t *testing.T) { + out, err := WithCredential("host=h user=old dbname=db", "vu", "pw") + if err != nil { + t.Fatal(err) + } + if u := mustUser(t, out); u != "vu" { + t.Fatalf("user = %q, want vu (old stripped)", u) + } + if pw := mustPassword(t, out); pw != "pw" { + t.Fatalf("password = %q, want pw", pw) + } +} + +// WithCredential's own round-trip verify guards escaping; a user and password full of +// quotes/backslashes must survive the keyword grammar. +func TestWithCredentialKeywordEscaping(t *testing.T) { + wantU, wantP := `v'u\x`, `p'w\y` + out, err := WithCredential("host=h dbname=db", wantU, wantP) + if err != nil { + t.Fatal(err) + } + if u := mustUser(t, out); u != wantU { + t.Fatalf("user = %q, want %q", u, wantU) + } + if pw := mustPassword(t, out); pw != wantP { + t.Fatalf("password = %q, want %q", pw, wantP) + } +} + +func TestWithCredentialURLSpecialCharsPassword(t *testing.T) { + tok := "p@ss:w/rd&x=1" + out, err := WithCredential("postgres://h:5432/db", "dynuser", tok) + if err != nil { + t.Fatal(err) + } + if u := mustUser(t, out); u != "dynuser" { + t.Fatalf("user = %q, want dynuser", u) + } + if pw := mustPassword(t, out); pw != tok { + t.Fatalf("password = %q, want %q", pw, tok) + } +} + +func TestHostPort(t *testing.T) { + cases := []struct { + dsn string + host string + port int + ok bool + }{ + {"postgres://u@127.0.0.1:6543/db", "127.0.0.1", 6543, true}, + {"postgres://u@db.corp/db", "db.corp", 5432, true}, // pgx defaults the port to 5432 + {"host=db.corp user=u", "db.corp", 5432, true}, // keyword form + {"host=/var/run/postgresql user=u", "", 0, false}, // unix-socket dir -> not ok + } + for _, c := range cases { + h, p, ok := HostPort(c.dsn) + if ok != c.ok || (ok && (h != c.host || p != c.port)) { + t.Errorf("HostPort(%q) = (%q,%d,%v), want (%q,%d,%v)", c.dsn, h, p, ok, c.host, c.port, c.ok) + } + } +} + +// HasPassword must reflect only an embedded password, not one pgx would resolve from +// ambient PGPASSWORD/~/.pgpass β€” otherwise a stale PGPASSWORD would short-circuit the +// whole ladder. +func TestHasPasswordIgnoresAmbientPGPASSWORD(t *testing.T) { + t.Setenv("PGPASSWORD", "from-env") + if HasPassword("postgres://bts@127.0.0.1:5432/db") { + t.Fatal("HasPassword must not report a password from ambient PGPASSWORD") + } + if !HasPassword("postgres://bts:embedded@127.0.0.1:5432/db") { + t.Fatal("HasPassword must detect an embedded URL password") + } + if !HasPassword("postgres://bts@127.0.0.1:5432/db?password=q") { + t.Fatal("HasPassword must detect a ?password= query param") + } +} + +// RedactPassword must not be fooled by an ambient PGPASSWORD (which pgx would fold in +// but which is never persisted) into refusing a password-less --pg-url at init. +func TestRedactPasswordIgnoresAmbientPGPASSWORD(t *testing.T) { + t.Setenv("PGPASSWORD", "from-env") + got, err := RedactPassword("postgres://bts@127.0.0.1:5432/db") + if err != nil { + t.Fatalf("RedactPassword must not be fooled by ambient PGPASSWORD: %v", err) + } + if HasPassword(got) { + t.Fatalf("redacted DSN still carries an embedded password: %q", got) + } +} diff --git a/internal/storage/pgdialect/credential_test.go b/internal/storage/pgdialect/credential_test.go new file mode 100644 index 000000000..92f46f28c --- /dev/null +++ b/internal/storage/pgdialect/credential_test.go @@ -0,0 +1,96 @@ +package pgdialect + +import ( + "testing" + + "github.com/jackc/pgx/v5" +) + +func TestHasPassword(t *testing.T) { + cases := []struct { + dsn string + want bool + }{ + {"postgres://bts:secret@127.0.0.1:5432/db", true}, + {"postgres://bts@127.0.0.1:5432/db", false}, + {"host=127.0.0.1 port=5432 user=bts password=secret dbname=db", true}, + {"host=127.0.0.1 port=5432 user=bts dbname=db", false}, + } + for _, c := range cases { + if got := HasPassword(c.dsn); got != c.want { + t.Errorf("HasPassword(%q) = %v, want %v", c.dsn, got, c.want) + } + } +} + +// mustPassword parses out and returns the password pgx sees. +func mustPassword(t *testing.T, out string) string { + t.Helper() + cfg, err := pgx.ParseConfig(out) + if err != nil { + t.Fatalf("pgx.ParseConfig(%q): %v", out, err) + } + return cfg.Password +} + +func TestWithPasswordURLForm(t *testing.T) { + out, err := WithPassword("postgres://bts@127.0.0.1:5432/db", "s3cr3t") + if err != nil { + t.Fatal(err) + } + if pw := mustPassword(t, out); pw != "s3cr3t" { + t.Fatalf("password = %q, want s3cr3t", pw) + } + // The username must be preserved. + cfg, _ := pgx.ParseConfig(out) + if cfg.User != "bts" { + t.Fatalf("user = %q, want bts", cfg.User) + } +} + +// A token with URL-reserved characters (as an RDS-IAM / GCP-IAM auth token carries) +// must round-trip through the URL grammar intact. +func TestWithPasswordURLSpecialChars(t *testing.T) { + tok := "X-Amz=abc&Signature=d/e+f%20g" + out, err := WithPassword("postgres://bts@host:5432/db", tok) + if err != nil { + t.Fatal(err) + } + if pw := mustPassword(t, out); pw != tok { + t.Fatalf("password = %q, want %q", pw, tok) + } +} + +// The keyword/value grammar is the case the old net/url merge silently dropped. +func TestWithPasswordKeywordForm(t *testing.T) { + out, err := WithPassword("host=127.0.0.1 port=5432 user=bts dbname=db", "s3 cr3t") // note the space + if err != nil { + t.Fatal(err) + } + if pw := mustPassword(t, out); pw != "s3 cr3t" { + t.Fatalf("password = %q, want %q", pw, "s3 cr3t") + } +} + +// An existing password in a keyword DSN is replaced, not duplicated. +func TestWithPasswordKeywordReplacesExisting(t *testing.T) { + out, err := WithPassword("host=h user=u password=old dbname=db", "new") + if err != nil { + t.Fatal(err) + } + if pw := mustPassword(t, out); pw != "new" { + t.Fatalf("password = %q, want new", pw) + } +} + +// A value with a single quote and a backslash survives keyword escaping. +func TestWithPasswordKeywordEscaping(t *testing.T) { + tok := `a'b\c` + out, err := WithPassword("host=h user=u dbname=db", tok) + if err != nil { + t.Fatal(err) + } + if pw := mustPassword(t, out); pw != tok { + t.Fatalf("password = %q, want %q", pw, tok) + } +} diff --git a/internal/storage/pgdialect/driver.go b/internal/storage/pgdialect/driver.go new file mode 100644 index 000000000..65ea6b06a --- /dev/null +++ b/internal/storage/pgdialect/driver.go @@ -0,0 +1,190 @@ +package pgdialect + +import ( + "context" + "database/sql" + "database/sql/driver" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" +) + +// Open returns a *sql.DB over pgx whose Prepare/PrepareContext translate bd's +// canonical (MySQL-dialect) SQL to Postgres via Translate. searchPath pins the +// workspace schema. +// +// This wrapper deliberately exposes no Queryer/Execer fast path, so database/sql +// routes every statement through Prepare β€” translation therefore happens in +// exactly one place, and an untranslatable construct fails loudly at PREPARE. +func Open(dsn, searchPath string) (*sql.DB, error) { + cfg, err := pgx.ParseConfig(dsn) + if err != nil { + // pgx's ParseConfigError leaves ?password= URL query params cleartext; scrub + // before the error can reach any log. + return nil, ScrubDSNError(dsn, err) + } + if searchPath != "" { + if cfg.RuntimeParams == nil { + cfg.RuntimeParams = map[string]string{} + } + cfg.RuntimeParams["search_path"] = searchPath + } + return sql.OpenDB(&connector{base: stdlib.GetConnector(*cfg)}), nil +} + +// OpenRaw returns a *sql.DB over pgx WITHOUT the translating connector wrapper, +// so SQL text reaches Postgres verbatim. It exists for schema DDL: the DDL is +// already native Postgres, and routing it through Translate mangles the +// $$-quoted function bodies and rewrites DDL constructs (e.g. now()). searchPath +// pins the workspace schema exactly as Open does. Callers must use native $n +// placeholders, not ?, since Translate is bypassed. +func OpenRaw(dsn, searchPath string) (*sql.DB, error) { + cfg, err := pgx.ParseConfig(dsn) + if err != nil { + // See Open: pgx does not redact query-param passwords in parse errors. + return nil, ScrubDSNError(dsn, err) + } + if searchPath != "" { + if cfg.RuntimeParams == nil { + cfg.RuntimeParams = map[string]string{} + } + cfg.RuntimeParams["search_path"] = searchPath + } + return sql.OpenDB(stdlib.GetConnector(*cfg)), nil +} + +type connector struct{ base driver.Connector } + +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + cn, err := c.base.Connect(ctx) + if err != nil { + return nil, err + } + return &conn{base: cn}, nil +} + +func (c *connector) Driver() driver.Driver { return c.base.Driver() } + +// conn wraps a pgx driver.Conn and translates SQL text at prepare time. +type conn struct{ base driver.Conn } + +func (c *conn) Prepare(query string) (driver.Stmt, error) { + q, err := Translate(query) + if err != nil { + return nil, err + } + base, err := c.base.Prepare(q) + if err != nil { + return nil, err + } + return &stmt{base: base, conn: c.base}, nil +} + +func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + q, err := Translate(query) + if err != nil { + return nil, err + } + var base driver.Stmt + if p, ok := c.base.(driver.ConnPrepareContext); ok { + base, err = p.PrepareContext(ctx, q) + } else { + base, err = c.base.Prepare(q) + } + if err != nil { + return nil, err + } + return &stmt{base: base, conn: c.base}, nil +} + +func (c *conn) Close() error { return c.base.Close() } + +//nolint:staticcheck // SA1019: driver.Conn requires the deprecated Begin; BeginTx is preferred and forwarded below. +func (c *conn) Begin() (driver.Tx, error) { return c.base.Begin() } + +func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if b, ok := c.base.(driver.ConnBeginTx); ok { + return b.BeginTx(ctx, opts) + } + return c.base.Begin() //nolint:staticcheck +} + +func (c *conn) ResetSession(ctx context.Context) error { + if rs, ok := c.base.(driver.SessionResetter); ok { + return rs.ResetSession(ctx) + } + return nil +} + +func (c *conn) IsValid() bool { + if v, ok := c.base.(driver.Validator); ok { + return v.IsValid() + } + return true +} + +func (c *conn) Ping(ctx context.Context) error { + if p, ok := c.base.(driver.Pinger); ok { + return p.Ping(ctx) + } + return nil +} + +// stmt wraps a pgx driver.Stmt so bd's MySQL TINYINT(1) discipline survives on +// Postgres: bd binds Go bool arguments to columns that are smallint here (not +// boolean), so CheckNamedValue coerces bool -> 0/1 before binding. Every other +// argument type is forwarded to pgx's own NamedValueChecker so pgx's rich type +// handling (arrays, time, etc.) is preserved. +type stmt struct { + base driver.Stmt + conn driver.Conn +} + +func (s *stmt) CheckNamedValue(nv *driver.NamedValue) error { + if b, ok := nv.Value.(bool); ok { + nv.Value = boolToInt64(b) + return nil + } + if chk, ok := s.conn.(driver.NamedValueChecker); ok { + return chk.CheckNamedValue(nv) + } + return driver.ErrSkip // fall back to database/sql's default conversion +} + +func (s *stmt) Close() error { return s.base.Close() } +func (s *stmt) NumInput() int { return s.base.NumInput() } + +//nolint:staticcheck // driver.Stmt requires the deprecated Exec; ExecContext is preferred and forwarded below. +func (s *stmt) Exec(args []driver.Value) (driver.Result, error) { return s.base.Exec(args) } + +//nolint:staticcheck // driver.Stmt requires the deprecated Query; QueryContext is preferred and forwarded below. +func (s *stmt) Query(args []driver.Value) (driver.Rows, error) { return s.base.Query(args) } + +func (s *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + if e, ok := s.base.(driver.StmtExecContext); ok { + return e.ExecContext(ctx, args) + } + return s.base.Exec(namedToValues(args)) //nolint:staticcheck +} + +func (s *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + if q, ok := s.base.(driver.StmtQueryContext); ok { + return q.QueryContext(ctx, args) + } + return s.base.Query(namedToValues(args)) //nolint:staticcheck +} + +func boolToInt64(b bool) int64 { + if b { + return 1 + } + return 0 +} + +func namedToValues(named []driver.NamedValue) []driver.Value { + vals := make([]driver.Value, len(named)) + for i, n := range named { + vals[i] = n.Value + } + return vals +} diff --git a/internal/storage/pgdialect/redact.go b/internal/storage/pgdialect/redact.go new file mode 100644 index 000000000..c415b2f56 --- /dev/null +++ b/internal/storage/pgdialect/redact.go @@ -0,0 +1,214 @@ +package pgdialect + +import ( + "errors" + "fmt" + "net/url" + "regexp" + "sort" + "strings" + + "github.com/jackc/pgx/v5" +) + +// dsnWSClass is the regexp character class spelling out every byte pgx treats as +// whitespace between libpq keyword/value tokens. Go's regexp \s is [\t\n\f\r ] and +// does NOT include \v (vertical tab); pgx.ParseConfig DOES accept \v as a token +// separator (confirmed: "host=h\vpassword=x" parses host="h", password="x"). Relying +// on \s here would silently fail to match a \v-separated password token at all, +// leaking it in full β€” so the class is spelled out explicitly instead. +const dsnWSClass = `\t\n\f\r \v` + +// kvPasswordRe matches a libpq keyword/value password token (password= or +// sslpassword=), whose value is either single-quoted or an unquoted run of +// non-whitespace characters that may itself contain backslash-escaped whitespace +// (pgx accepts password=SUPER\ SECRET as the single value `SUPER\ SECRET`, backslash +// retained literally β€” confirmed against pgx.ParseConfig). +var kvPasswordRe = regexp.MustCompile(`(?i)(^|[` + dsnWSClass + `])(?:password|sslpassword)[` + dsnWSClass + `]*=[` + dsnWSClass + `]*(?:'(?:[^'\\]|\\.)*'|(?:[^` + dsnWSClass + `\\]|\\.)*)`) + +// pwValueRe captures the VALUE of a libpq keyword/value password token: group 1 is +// the single-quoted body, group 2 is an unquoted run of non-whitespace characters, +// including any backslash-escaped whitespace sequences (see kvPasswordRe). +var pwValueRe = regexp.MustCompile(`(?i)(?:^|[` + dsnWSClass + `])(?:password|sslpassword)[` + dsnWSClass + `]*=[` + dsnWSClass + `]*(?:'((?:[^'\\]|\\.)*)'|((?:[^` + dsnWSClass + `\\]|\\.)+))`) + +// RedactPassword returns dsn with the password removed, or an error if a password +// cannot be safely removed. It strips every known password location β€” URL userinfo, +// URL query params (password, sslpassword), and libpq keyword/value tokens β€” then +// VERIFIES that no EMBEDDED password survives, failing closed rather than persisting a +// secret in a DSN shape we did not anticipate. The check is embedded-only (HasPassword, +// not pgx.ParseConfig.Password) so an ambient PGPASSWORD/~/.pgpass β€” which pgx would +// fold in but which is never written to disk β€” does not falsely block init. +// +// Callers persist the returned string (e.g. to metadata.json) and re-supply the +// password at open time via the credential ladder (see the backend's resolveDSNCredential), +// so nothing operational is lost by never writing the password to disk. The hard +// invariant β€” a password must never be persisted β€” is enforced here, not assumed. +func RedactPassword(dsn string) (string, error) { + stripped := stripPasswordBestEffort(dsn) + // It must still parse, or we cannot reason about it β€” refuse a shape we mangled. + if _, err := pgx.ParseConfig(stripped); err != nil { + return "", fmt.Errorf("cannot verify the connection string is free of a password (unparseable after redaction): %w", err) + } + if HasPassword(stripped) { + return "", fmt.Errorf("refusing to persist a connection string that still carries a password; supply it via BEADS_PG_PASSWORD instead of embedding it in --pg-url") + } + return stripped, nil +} + +// stripPasswordBestEffort removes passwords from the two connection-string shapes +// pgx accepts. The authoritative guarantee is RedactPassword's post-parse check; this +// only has to handle the common forms well enough not to trip that check falsely. +func stripPasswordBestEffort(dsn string) string { + // URL form (scheme://...): strip userinfo password and query password/sslpassword. + if u, err := url.Parse(dsn); err == nil && u.Scheme != "" { + if u.User != nil { + if name := u.User.Username(); name == "" { + u.User = nil + } else { + u.User = url.User(name) + } + } + q := u.Query() + q.Del("password") + q.Del("sslpassword") + u.RawQuery = q.Encode() + return u.String() + } + // libpq keyword/value form (host=... password=... ...): drop the password tokens + // and collapse surrounding whitespace so the remainder still parses. + out := kvPasswordRe.ReplaceAllString(dsn, "$1") + return strings.TrimSpace(strings.Join(strings.Fields(out), " ")) +} + +// ScrubDSNString returns s with every cleartext password embedded in dsn replaced +// by "xxxxx". It is the string-level primitive behind ScrubDSNError, exposed so a +// caller that echoes a DSN somewhere other than an error β€” a telemetry span, a log +// line β€” can redact it through the SAME parser-backed extraction (URL userinfo, URL +// query password/sslpassword, and libpq keyword/value tokens) instead of a separate +// hand-rolled scan that misses a form. dsn and s are usually the same string (scrub a +// DSN in place); they differ only when the password leaked into surrounding text, as +// ScrubDSNError passes an error message as s. An empty dsn, or one with no password, +// returns s unchanged. +func ScrubDSNString(dsn, s string) string { + secrets := dsnPasswordValues(dsn) + // Replace the longest secrets first. dsnPasswordValues can return one secret that + // is a byte-for-byte prefix of another (e.g. a userinfo password "foo" and a query + // password "fooACTUAL"); replacing the short one first would turn "fooACTUAL" into + // "xxxxxACTUAL" before the long-secret pass ever runs, leaking the "ACTUAL" tail + // since ReplaceAll can no longer find the (now-mangled) longer substring. + sort.Slice(secrets, func(i, j int) bool { return len(secrets[i]) > len(secrets[j]) }) + for _, secret := range secrets { + s = strings.ReplaceAll(s, secret, "xxxxx") + } + return s +} + +// ScrubDSNError returns a new error whose message has every cleartext password +// embedded in dsn removed. pgx's ParseConfigError redacts only URL userinfo, so a +// `?password=`/`?sslpassword=` URL query param β€” the shape `bd init` connects with β€” +// otherwise survives verbatim in the error text and leaks the secret to logs on any +// parse or TLS-config failure. The returned error intentionally does NOT wrap err: +// the pgx error's own Error() still contains the secret, so exposing it through the +// unwrap chain would defeat the redaction. Non-nil in, non-nil out; nil passes through. +func ScrubDSNError(dsn string, err error) error { + if err == nil { + return nil + } + return errors.New(ScrubDSNString(dsn, err.Error())) +} + +// isPasswordKey reports whether k names one of the URL query password params +// (case-insensitively). +func isPasswordKey(k string) bool { + switch strings.ToLower(k) { + case "password", "sslpassword": + return true + } + return false +} + +// dsnPasswordValues returns every cleartext password embedded in dsn β€” URL userinfo, +// URL query params (password, sslpassword), and libpq keyword/value tokens β€” in both +// the raw (as-written) and percent-decoded forms, so a value matches however the +// connection string is echoed back. Empty values are skipped, and results are +// de-duplicated. +func dsnPasswordValues(dsn string) []string { + seen := map[string]bool{} + var vals []string + add := func(v string) { + if v == "" || seen[v] { + return + } + seen[v] = true + vals = append(vals, v) + } + if u, err := url.Parse(dsn); err == nil && u.Scheme != "" { + if u.User != nil { + if pw, ok := u.User.Password(); ok { + // url.Parse percent-decodes userinfo, so u.User.Password() is only + // the decoded value. pgx and telemetry sinks echo the DSN verbatim, + // so also scrub the raw as-written form β€” mirroring the query-param + // branch below, which adds both raw and decoded values. + add(pw) + add(rawURLUserinfoPassword(dsn)) + } + } + // pgx echoes the raw query string back verbatim, so match the raw value; + // also add the decoded value in case a caller logs the parsed form. + for _, pair := range strings.Split(u.RawQuery, "&") { + key, val, ok := strings.Cut(pair, "=") + if !ok { + continue + } + // url.Query() (which pgx.ParseConfig uses) percent-decodes query KEYS as + // well as values, so a query key of pass%77ord or %70assword is a live + // "password" param even though the raw key string doesn't say so. Compare + // both the raw (as-written) key and its unescaped form β€” ignoring an + // unescape error and falling back to the raw compare, which already ran. + matched := isPasswordKey(key) + if !matched { + if dk, err := url.QueryUnescape(key); err == nil { + matched = isPasswordKey(dk) + } + } + if matched { + add(val) + if dec, err := url.QueryUnescape(val); err == nil { + add(dec) + } + } + } + } + for _, m := range pwValueRe.FindAllStringSubmatch(dsn, -1) { + add(m[1]) // single-quoted body + add(m[2]) // unquoted token + } + return vals +} + +// rawURLUserinfoPassword returns the userinfo password of a URL-form DSN exactly +// as written β€” still percent-encoded β€” or "" when there is none. url.Parse decodes +// userinfo, so an encoded secret like SUPER%2ASECRET survives verbatim when a sink +// echoes the raw DSN; extracting it structurally here (rather than re-encoding the +// decoded value) captures the original bytes regardless of encoding form. Mirrors +// the authority split url.Parse performs: password is after the first ':' of the +// userinfo, which is everything before the last '@' of the authority. +func rawURLUserinfoPassword(dsn string) string { + i := strings.Index(dsn, "://") + if i < 0 { + return "" + } + authority := dsn[i+len("://"):] + if j := strings.IndexAny(authority, "/?#"); j >= 0 { + authority = authority[:j] + } + at := strings.LastIndex(authority, "@") + if at < 0 { + return "" + } + _, pw, ok := strings.Cut(authority[:at], ":") + if !ok { + return "" + } + return pw +} diff --git a/internal/storage/pgdialect/redact_error_test.go b/internal/storage/pgdialect/redact_error_test.go new file mode 100644 index 000000000..4260e6039 --- /dev/null +++ b/internal/storage/pgdialect/redact_error_test.go @@ -0,0 +1,188 @@ +package pgdialect + +import ( + "database/sql" + "errors" + "strings" + "testing" +) + +// TestScrubDSNError proves an error carrying a DSN can never echo the cleartext +// password back. pgx already redacts URL userinfo and libpq keyword passwords, so +// the URL query-param vectors are the ones that actually leak (see +// TestOpenRedactsPasswordInError); this drives every shape against a worst-case +// error that echoes the raw DSN verbatim. +func TestScrubDSNError(t *testing.T) { + const secret = "SUPERSECRET" + leakForms := []string{secret, "SUPER%2ASECRET", "SUPER*SECRET"} + cases := []struct { + name string + dsn string + }{ + {"url query param", "postgres://u@h:5432/db?password=" + secret}, + {"url query sslpassword", "postgres://u@h:5432/db?sslpassword=" + secret + "&sslmode=require"}, + {"url query param encoded", "postgres://u@h:5432/db?password=SUPER%2ASECRET"}, + {"url userinfo", "postgres://u:" + secret + "@h:5432/db"}, + {"url userinfo encoded", "postgres://u:SUPER%2ASECRET@h:5432/db"}, + {"libpq keyword", "host=h user=u password=" + secret + " dbname=db"}, + {"libpq keyword quoted", "host=h user=u password='" + secret + "' dbname=db"}, + {"libpq sslpassword", "host=h user=u sslpassword=" + secret + " dbname=db"}, + {"url query param percent-encoded key (pass%77ord)", "postgres://u@h:5432/db?pass%77ord=" + secret}, + {"url query sslpassword percent-encoded key (sslpass%77ord)", "postgres://u@h:5432/db?sslpass%77ord=" + secret + "&sslmode=require"}, + {"url query param fully percent-encoded key (%70assword)", "postgres://u@h:5432/db?%70assword=" + secret}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + raw := errors.New("cannot parse `" + tc.dsn + "`: boom") + got := ScrubDSNError(tc.dsn, raw).Error() + for _, form := range leakForms { + if strings.Contains(got, form) { + t.Errorf("scrubbed error still leaks password (%q): %q", form, got) + } + } + // The non-secret failure reason must survive so the error stays useful. + if !strings.Contains(got, "boom") { + t.Errorf("scrubbed error dropped the failure reason: %q", got) + } + }) + } + if ScrubDSNError("postgres://u@h/db?password=x", nil) != nil { + t.Error("ScrubDSNError(dsn, nil) must return nil") + } +} + +// TestScrubDSNString proves the string-level primitive removes every password form +// when a DSN is echoed as plain text (a telemetry span, a log line) rather than an +// error β€” the same shapes TestScrubDSNError covers, but exercised directly since the +// telemetry scrubber depends on this function, not on ScrubDSNError. +func TestScrubDSNString(t *testing.T) { + const secret = "SUPERSECRET" + leakForms := []string{secret, "SUPER%2ASECRET", "SUPER*SECRET"} + cases := []struct { + name string + dsn string + }{ + {"url userinfo", "postgres://u:" + secret + "@h:5432/db"}, + {"url userinfo encoded", "postgres://u:SUPER%2ASECRET@h:5432/db"}, + {"url query param", "postgres://u@h:5432/db?password=" + secret}, + {"url sslpassword", "postgres://u@h:5432/db?sslpassword=" + secret + "&sslmode=require"}, + {"libpq keyword", "host=h user=u password=" + secret + " dbname=db"}, + {"libpq sslpassword", "host=h user=u sslpassword=" + secret + " dbname=db"}, + {"url query param percent-encoded key (pass%77ord)", "postgres://u@h:5432/db?pass%77ord=" + secret}, + {"url query sslpassword percent-encoded key (sslpass%77ord)", "postgres://u@h:5432/db?sslpass%77ord=" + secret + "&sslmode=require"}, + {"url query param fully percent-encoded key (%70assword)", "postgres://u@h:5432/db?%70assword=" + secret}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := ScrubDSNString(tc.dsn, tc.dsn) + for _, form := range leakForms { + if strings.Contains(got, form) { + t.Errorf("ScrubDSNString(%q) still leaks password (%q): %q", tc.dsn, form, got) + } + } + if !strings.Contains(got, "xxxxx") { + t.Errorf("ScrubDSNString(%q) did not redact: %q", tc.dsn, got) + } + }) + } + // A DSN with no password, and an empty DSN, must pass the target string through. + if got := ScrubDSNString("postgres://u@h:5432/db", "postgres://u@h:5432/db"); got != "postgres://u@h:5432/db" { + t.Errorf("passwordless DSN altered: %q", got) + } + if got := ScrubDSNString("", "no dsn here"); got != "no dsn here" { + t.Errorf("empty dsn altered s: %q", got) + } +} + +// TestOpenRedactsPasswordInError drives the real pgx parser: a DSN that fails to +// parse while carrying a ?password=/?sslpassword= query param must not leak the +// secret through Open or OpenRaw. connect_timeout=x is what makes ParseConfig fail; +// the query-param password is exactly the vector pgx's own redaction misses. +func TestOpenRedactsPasswordInError(t *testing.T) { + const secret = "SUPERSECRET" + dsns := []string{ + "postgres://u@h:5432/db?password=" + secret + "&connect_timeout=x", + "postgres://u@h:5432/db?sslpassword=" + secret + "&connect_timeout=x", + } + opens := []struct { + name string + open func(string, string) (*sql.DB, error) + }{ + {"Open", Open}, + {"OpenRaw", OpenRaw}, + } + for _, dsn := range dsns { + for _, o := range opens { + t.Run(o.name+" "+dsn, func(t *testing.T) { + db, err := o.open(dsn, "public") + if err == nil { + _ = db.Close() + t.Fatal("expected a parse error, got nil") + } + if strings.Contains(err.Error(), secret) { + t.Errorf("%s leaked password in error: %q", o.name, err.Error()) + } + }) + } + } +} + +// TestScrubDSNStringEscapedWhitespaceValue proves an unquoted libpq keyword/value +// password containing a backslash-escaped space is redacted in full. pgx accepts +// password=SUPER\ SECRET as the single password `SUPER\ SECRET` (the backslash is +// retained literally, the escaped space is part of the value β€” confirmed against +// pgx.ParseConfig); the pre-fix value regex (a \S+-shaped run) stopped at the +// unescaped whitespace boundary and left the tail " SECRET" behind in plaintext. +func TestScrubDSNStringEscapedWhitespaceValue(t *testing.T) { + dsn := `host=h user=u password=SUPER\ SECRET dbname=db` + got := ScrubDSNString(dsn, dsn) + if strings.Contains(got, "SECRET") { + t.Errorf("ScrubDSNString(%q) leaked escaped-whitespace password tail: %q", dsn, got) + } + if strings.Contains(got, `SUPER\`) { + t.Errorf("ScrubDSNString(%q) leaked escaped-whitespace password head: %q", dsn, got) + } + if !strings.Contains(got, "xxxxx") { + t.Errorf("ScrubDSNString(%q) did not redact: %q", dsn, got) + } + if !strings.Contains(got, "host=h") || !strings.Contains(got, "dbname=db") { + t.Errorf("ScrubDSNString(%q) mangled non-secret structure: %q", dsn, got) + } +} + +// TestScrubDSNStringVerticalTabSeparator proves a password separated from the +// preceding keyword/value token by a vertical tab (\v) is still found and redacted. +// pgx treats \v as inter-token whitespace in libpq keyword/value DSNs (confirmed +// against pgx.ParseConfig: "host=h\vpassword=x" parses host="h", password="x"), but +// Go's regexp \s class is [\t\n\f\r ] and does NOT include \v, so the pre-fix +// password regex β€” anchored on (^|\s) β€” never matched at a \v boundary and the +// password leaked entirely. +func TestScrubDSNStringVerticalTabSeparator(t *testing.T) { + const secret = "SUPERSECRET" + dsn := "host=h user=u" + "\v" + "password=" + secret + " dbname=db" + got := ScrubDSNString(dsn, dsn) + if strings.Contains(got, secret) { + t.Errorf("ScrubDSNString(%q) leaked password separated by a vertical tab: %q", dsn, got) + } + if !strings.Contains(got, "xxxxx") { + t.Errorf("ScrubDSNString(%q) did not redact vertical-tab-separated password: %q", dsn, got) + } +} + +// TestScrubDSNStringOverlappingSecretsPrefix proves that when one collected secret is +// a byte-for-byte prefix of another (userinfo password "foo", query password +// "fooACTUAL"), redaction does not leak the longer secret's non-overlapping tail. +// ScrubDSNString applies strings.ReplaceAll sequentially per collected secret; doing +// the short secret ("foo") first turns "fooACTUAL" into "xxxxxACTUAL" before the +// long-secret pass ever runs, so the second pass finds no "fooACTUAL" substring left +// to replace and "ACTUAL" survives in plaintext. +func TestScrubDSNStringOverlappingSecretsPrefix(t *testing.T) { + dsn := "postgres://u:foo@h:5432/db?password=fooACTUAL" + got := ScrubDSNString(dsn, dsn) + if strings.Contains(got, "ACTUAL") { + t.Errorf("ScrubDSNString(%q) leaked the tail of a secret that is a prefix of another secret: %q", dsn, got) + } + if strings.Contains(got, ":foo@") { + t.Errorf("ScrubDSNString(%q) did not redact the shorter overlapping secret: %q", dsn, got) + } +} diff --git a/internal/storage/pgdialect/redact_matrix_test.go b/internal/storage/pgdialect/redact_matrix_test.go new file mode 100644 index 000000000..22fce592e --- /dev/null +++ b/internal/storage/pgdialect/redact_matrix_test.go @@ -0,0 +1,179 @@ +package pgdialect + +import ( + "testing" + + "github.com/jackc/pgx/v5" +) + +// placeholderPW is an obvious non-secret stand-in used wherever a password appears in +// these tests, so nothing here can be mistaken for a leaked real credential. +const placeholderPW = "s3cr3t-PLACEHOLDER" + +// tlsExpect encodes what the redacted DSN's TLS state must be: pgx folds sslmode into +// TLSConfig (sslmode=disable -> nil, sslmode=require -> non-nil), so this is how we +// prove sslmode survived redaction. +type tlsExpect int + +const ( + tlsSkip tlsExpect = iota // don't assert (case sets no sslmode) + tlsMustNil // sslmode=disable -> TLSConfig must be nil + tlsMustSet // sslmode=require -> TLSConfig must be non-nil +) + +// TestRedactPasswordMatrix locks the security invariant across every secret-bearing DSN +// shape pgx accepts. For each case it asserts BOTH: +// +// (a) the persisted string, re-parsed by pgx β€” the SAME parser that opens the +// connection β€” carries NO password (prove the secret is gone, not just absent +// from the raw string); and +// (b) every non-secret component (host, port, database, user, sslmode, other query +// params) survives redaction. +// +// A no-password DSN must additionally round-trip byte-for-byte unchanged. +func TestRedactPasswordMatrix(t *testing.T) { + cases := []struct { + name string + dsn string + wantHost string + wantPort uint16 + wantDB string + wantUser string + wantTLS tlsExpect + wantParams map[string]string // RuntimeParams that must be preserved + roundTripSame bool // no-password input must come back unchanged + // skipUser: when the input has no user, pgx defaults cfg.User to the OS user, so + // the parsed user is environment-dependent and not something redaction controls. + skipUser bool + }{ + { + name: "url userinfo password", + dsn: "postgres://bts:" + placeholderPW + "@127.0.0.1:5432/db", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", + }, + { + // The special characters are percent-encoded in userinfo per RFC 3986: + // '@'=%40, ':'=%3A, '/'=%2F. The redactor must still strip it cleanly. + name: "url userinfo password with special chars (url-encoded)", + dsn: "postgres://bts:p%40ss%3Aw%2Frd-" + placeholderPW + "@127.0.0.1:5432/db", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", + }, + { + name: "url query param password + sslmode preserved", + dsn: "postgres://bts@127.0.0.1:5432/db?password=" + placeholderPW + "&sslmode=disable", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", + wantTLS: tlsMustNil, + }, + { + name: "url sslpassword query param + sslmode preserved", + dsn: "postgres://bts@127.0.0.1:5432/db?sslpassword=" + placeholderPW + "&sslmode=require", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", + wantTLS: tlsMustSet, + }, + { + name: "url userinfo password with sslmode and application_name preserved", + dsn: "postgres://bts:" + placeholderPW + "@127.0.0.1:5432/db?sslmode=require&application_name=beads", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", + wantTLS: tlsMustSet, wantParams: map[string]string{"application_name": "beads"}, + }, + { + name: "url password but no user", + dsn: "postgres://:" + placeholderPW + "@127.0.0.1:5432/db", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", skipUser: true, + }, + { + name: "keyword/value password", + dsn: "host=127.0.0.1 port=5432 user=bts password=" + placeholderPW + " dbname=db", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", + }, + { + name: "keyword/value single-quoted password containing spaces", + dsn: "host=127.0.0.1 port=5432 user=bts password='se cret " + placeholderPW + "' dbname=db", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", + }, + { + name: "keyword/value sslpassword + sslmode preserved", + dsn: "host=127.0.0.1 port=5432 user=bts sslpassword=" + placeholderPW + " sslmode=require dbname=db", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", + wantTLS: tlsMustSet, + }, + { + name: "keyword/value password but no user", + dsn: "host=127.0.0.1 port=5432 password=" + placeholderPW + " dbname=db", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", skipUser: true, + }, + { + // Params are in sorted key order because stripPasswordBestEffort re-encodes + // the query via url.Values.Encode(), which sorts keys; the byte-for-byte + // round-trip invariant is asserted against a DSN already in that canonical + // order (application_name < sslmode). + name: "no password (url) round-trips unchanged", + dsn: "postgres://bts@127.0.0.1:5432/db?application_name=beads&sslmode=require", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", + wantTLS: tlsMustSet, wantParams: map[string]string{"application_name": "beads"}, + roundTripSame: true, + }, + { + name: "no password (keyword/value) round-trips unchanged", + dsn: "host=127.0.0.1 port=5432 user=bts dbname=db sslmode=require", + wantHost: "127.0.0.1", wantPort: 5432, wantDB: "db", wantUser: "bts", + wantTLS: tlsMustSet, roundTripSame: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := RedactPassword(tc.dsn) + if err != nil { + t.Fatalf("RedactPassword(%q) errored: %v", tc.dsn, err) + } + + // (a) The authoritative security check: the same parser pgx uses to open the + // connection must find no password in the persisted string. + cfg, perr := pgx.ParseConfig(got) + if perr != nil { + t.Fatalf("redacted DSN no longer parses: %q: %v", got, perr) + } + if cfg.Password != "" { + t.Fatalf("PASSWORD LEAK: RedactPassword(%q) = %q still carries password %q", tc.dsn, got, cfg.Password) + } + // Belt and suspenders: the embedded-only detector must also agree. + if HasPassword(got) { + t.Fatalf("PASSWORD LEAK: HasPassword still true for redacted %q", got) + } + + // (b) Non-secret components must survive. + if cfg.Host != tc.wantHost { + t.Errorf("host = %q, want %q", cfg.Host, tc.wantHost) + } + if cfg.Port != tc.wantPort { + t.Errorf("port = %d, want %d", cfg.Port, tc.wantPort) + } + if cfg.Database != tc.wantDB { + t.Errorf("database = %q, want %q", cfg.Database, tc.wantDB) + } + if !tc.skipUser && cfg.User != tc.wantUser { + t.Errorf("user = %q, want %q", cfg.User, tc.wantUser) + } + switch tc.wantTLS { + case tlsMustNil: + if cfg.TLSConfig != nil { + t.Errorf("sslmode not preserved: TLSConfig = %v, want nil (sslmode=disable)", cfg.TLSConfig) + } + case tlsMustSet: + if cfg.TLSConfig == nil { + t.Errorf("sslmode not preserved: TLSConfig = nil, want non-nil (sslmode=require)") + } + } + for k, want := range tc.wantParams { + if got := cfg.RuntimeParams[k]; got != want { + t.Errorf("param %q = %q, want %q (must be preserved)", k, got, want) + } + } + + if tc.roundTripSame && got != tc.dsn { + t.Errorf("no-password DSN must round-trip unchanged:\n in = %q\n out = %q", tc.dsn, got) + } + }) + } +} diff --git a/internal/storage/pgdialect/redact_test.go b/internal/storage/pgdialect/redact_test.go new file mode 100644 index 000000000..bc07f6a05 --- /dev/null +++ b/internal/storage/pgdialect/redact_test.go @@ -0,0 +1,58 @@ +package pgdialect + +import ( + "testing" + + "github.com/jackc/pgx/v5" +) + +// TestRedactPassword encodes the hard invariant: for every DSN shape pgx accepts, +// RedactPassword's output must carry NO password (verified by re-parsing with pgx), +// while the rest of the connection string survives. These cases include the two +// vectors the red-team confirmed leaked: a query-param password and a libpq +// keyword/value password. +func TestRedactPassword(t *testing.T) { + cases := []struct { + name string + dsn string + }{ + {"url userinfo", "postgres://bts:secret@127.0.0.1:5432/db"}, + {"url query param", "postgres://bts@127.0.0.1:5432/db?password=secret"}, + {"url query param + other", "postgres://bts@127.0.0.1:5432/db?password=secret&sslmode=disable"}, + {"url sslpassword", "postgres://bts@127.0.0.1:5432/db?sslpassword=secret&sslmode=require"}, + {"libpq kv", "host=127.0.0.1 port=5432 user=bts password=secret dbname=db"}, + {"libpq kv quoted", "host=127.0.0.1 user=bts password='se cret' dbname=db"}, + {"libpq kv sslpassword", "host=127.0.0.1 user=bts sslpassword=secret dbname=db"}, + {"no password (url)", "postgres://bts@127.0.0.1:5432/db"}, + {"no password (kv)", "host=127.0.0.1 user=bts dbname=db"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := RedactPassword(tc.dsn) + if err != nil { + t.Fatalf("RedactPassword(%q) errored: %v", tc.dsn, err) + } + // The authoritative check: pgx β€” the same parser that opens the + // connection β€” must see no password in the persisted string. + cfg, perr := pgx.ParseConfig(got) + if perr != nil { + t.Fatalf("redacted DSN no longer parses: %q: %v", got, perr) + } + if cfg.Password != "" { + t.Fatalf("PASSWORD LEAK: RedactPassword(%q) = %q still carries password %q", tc.dsn, got, cfg.Password) + } + }) + } +} + +// TestRedactPasswordFailsClosed: if the input still has a password that the +// best-effort strip missed, RedactPassword must ERROR rather than return a leaky +// string. We simulate an "unanticipated shape" by asserting the post-parse guard is +// what enforces this β€” a KV password with an exotic separator that the regex could +// miss must never silently pass through with a password. +func TestRedactPasswordNeverReturnsASecret(t *testing.T) { + // Sanity: a plain non-DSN string must not be returned as a "clean" DSN. + if _, err := RedactPassword("::::not a url"); err == nil { + t.Fatal("expected error for an unparseable connection string, got nil") + } +} diff --git a/internal/storage/pgdialect/translate.go b/internal/storage/pgdialect/translate.go new file mode 100644 index 000000000..3605be3ac --- /dev/null +++ b/internal/storage/pgdialect/translate.go @@ -0,0 +1,777 @@ +// Package pgdialect translates the MySQL-dialect SQL that bd's shared storage +// layer (internal/storage/sqlbuild and internal/storage/issueops) emits into +// PostgreSQL, so that same layer can run unchanged over a pgx-backed +// database/sql connection. +// +// It deliberately handles only the constructs bd actually emits on the storage +// path β€” a finite, enumerable set (see corpus_test.go). Anything it does not +// recognize passes through untouched and surfaces as a loud Postgres syntax +// error at PREPARE time, never a silent mistranslation. That "fail loud" +// property is the whole safety argument: a divergence is a build failure, not a +// subtly wrong row set. +package pgdialect + +import ( + "fmt" + "strings" +) + +// Translate rewrites a single MySQL statement into its Postgres equivalent. +// +// Pass order is deliberate. Function-shaped rewrites (IF, VALUES, CONCAT, +// LOCATE, JSON) run first so the ON DUPLICATE KEY body is already Postgres by +// the time it is wrapped into ON CONFLICT. Placeholder numbering runs last, so +// every earlier pass can move text freely; none of them introduce a literal +// `?`, so left-to-right numbering stays correct. +func Translate(sql string) (string, error) { + out := sql + // CONCAT/LOCATE are unambiguous functions anywhere they appear. IF() and + // VALUES() are NOT translated globally: VALUES is also the INSERT keyword, + // and both only appear as functions inside an ON DUPLICATE KEY UPDATE body, + // so they are translated there (rewriteOnDuplicateKey). + out = rewriteFuncCalls(out, "CONCAT", concatArgs) // CONCAT(a,b) -> (a || b) + out = rewriteFuncCalls(out, "LOCATE", locateArgs) // LOCATE(n,h) -> POSITION(n IN h) + out = rewriteFuncCalls(out, "INSTR", instrArgs) // INSTR(h,n) -> POSITION(n IN h) (args reversed vs LOCATE) + out = rewriteFuncCalls(out, "DATE_FORMAT", dateFormatArgs) // DATE_FORMAT(t,'%Y-%m-%dT%H:%i:%sZ') -> to_char(t,'YYYY-MM-DD"T"HH24:MI:SS"Z"') + out = rewriteFuncCalls(out, "JSON_OBJECT", jsonObjectArgs) // JSON_OBJECT(k,v,...) -> jsonb_build_object(k,v,...) + out = rewriteCastChar(out) // CAST(x AS CHAR[(n)]) -> (x)::text + // CURRENT_TIMESTAMP (not now()) so the replacement contains no token that a + // later pass would re-match. AT TIME ZONE 'utc' yields naive-UTC to match + // bd's Dolt storage discipline. + out = replaceIdentAll(out, "UTC_TIMESTAMP()", "(CURRENT_TIMESTAMP AT TIME ZONE 'utc')") + out = replaceIdentAll(out, "NOW()", "(CURRENT_TIMESTAMP AT TIME ZONE 'utc')") + out = rewriteLiteralJSONUnquote(out) + out = rewriteDynamicJSONPath(out) + out = rewriteLabelOrderBy(out) + out = rewriteInsertIgnore(out) + out = rewriteReplaceInto(out) + out = rewriteOnDuplicateKey(out) + out = rewriteUpdateSetTargets(out) + out = rewriteBackticks(out) + out = numberPlaceholders(out) + return out, nil +} + +// codeMask returns a per-byte bitmap: mask[i] is true iff byte i lies outside +// any string literal or comment. bd uses single-quoted strings (” escapes a +// quote), `--` line comments, and `/* */` blocks; it never double-quotes +// strings (those are identifiers in Postgres/ANSI mode), so a double quote is +// always code. Every text rewrite consults this so it never touches quoted data +// or commented-out SQL. +func codeMask(sql string) []bool { + b := []byte(sql) + m := make([]bool, len(b)) + var inStr, inLine, inBlock bool + for i := 0; i < len(b); i++ { + c := b[i] + switch { + case inLine: + if c == '\n' { + inLine = false + } + case inBlock: + if c == '*' && i+1 < len(b) && b[i+1] == '/' { + i++ // consume '/' + inBlock = false + } + case inStr: + if c == '\'' { + if i+1 < len(b) && b[i+1] == '\'' { + i++ // escaped quote: consume both + } else { + inStr = false + } + } + default: + switch { + case c == '\'': + inStr = true + case c == '-' && i+1 < len(b) && b[i+1] == '-': + i++ + inLine = true + case c == '/' && i+1 < len(b) && b[i+1] == '*': + i++ + inBlock = true + default: + m[i] = true + } + } + } + return m +} + +func isIdentByte(c byte) bool { + return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') +} + +// numberPlaceholders replaces each code-region `?` with $1, $2, … in order. +func numberPlaceholders(sql string) string { + m := codeMask(sql) + var b strings.Builder + n := 0 + for i := 0; i < len(sql); i++ { + if m[i] && sql[i] == '?' { + n++ + fmt.Fprintf(&b, "$%d", n) + continue + } + b.WriteByte(sql[i]) + } + return b.String() +} + +// rewriteBackticks turns `ident` into "ident" in code regions. bd only uses +// backticks around reserved-word column names (e.g. `key`). +func rewriteBackticks(sql string) string { + m := codeMask(sql) + out := []byte(sql) + for i := range out { + if m[i] && out[i] == '`' { + out[i] = '"' + } + } + return string(out) +} + +// replaceIdentAll replaces every code-region occurrence of a literal token +// (case-insensitive) with repl. Used for parameterless functions like +// UTC_TIMESTAMP() and NOW(). +func replaceIdentAll(sql, token, repl string) string { + lower := strings.ToLower(token) + m := codeMask(sql) + var b strings.Builder + // Single forward pass: never re-scans the inserted replacement, so a repl + // that contains the token (NOW() -> "(now() …)") can't loop. + for i := 0; i < len(sql); { + if m[i] && i+len(token) <= len(sql) && strings.ToLower(sql[i:i+len(token)]) == lower { + b.WriteString(repl) + i += len(token) + continue + } + b.WriteByte(sql[i]) + i++ + } + return b.String() +} + +// rewriteFuncCalls finds NAME(args…) in code regions (balanced parens, +// string-aware) and replaces each with rebuild(args). It recurses on each +// argument first, so nested calls (LOCATE(CONCAT(…), x)) translate correctly. +func rewriteFuncCalls(sql, name string, rebuild func(args []string) string) string { + lower := strings.ToLower(name) + searchFrom := 0 + for { + idx := findCallStart(sql, lower, searchFrom) + if idx < 0 { + return sql + } + args, end, ok := splitCallArgs(sql, idx+len(name)) + if !ok { + searchFrom = idx + len(name) // not a call; skip past this token + continue + } + for i := range args { + args[i] = rewriteFuncCalls(strings.TrimSpace(args[i]), name, rebuild) + } + rebuilt := rebuild(args) + sql = sql[:idx] + rebuilt + sql[end:] + // Advance past the rebuilt region β€” nested same-name calls were already + // handled by the recursion on args, so we never need to revisit it. This + // guarantees forward progress (termination) even if a rebuild happened to + // contain the function name. + searchFrom = idx + len(rebuilt) + } +} + +// findCallStart returns the index of the next code-region, word-boundaried +// occurrence of name (case-insensitive) that is immediately followed by `(` +// (optionally after spaces), at or after `from`; or -1. +func findCallStart(sql, lowerName string, from int) int { + m := codeMask(sql) + for i := from; i+len(lowerName) <= len(sql); i++ { + if !m[i] { + continue + } + if strings.ToLower(sql[i:i+len(lowerName)]) != lowerName { + continue + } + if i > 0 && isIdentByte(sql[i-1]) { + continue // not a word boundary + } + j := i + len(lowerName) + for j < len(sql) && (sql[j] == ' ' || sql[j] == '\t' || sql[j] == '\n') { + j++ + } + if j < len(sql) && sql[j] == '(' { + return i + } + } + return -1 +} + +// splitCallArgs, given the index just after a function name, finds the opening +// paren, walks to the matching close (string-aware), and returns the top-level +// comma-separated arguments plus the index just past the close paren. +func splitCallArgs(sql string, afterName int) (args []string, end int, ok bool) { + m := codeMask(sql) + i := afterName + for i < len(sql) && sql[i] != '(' { + if sql[i] != ' ' && sql[i] != '\t' && sql[i] != '\n' { + return nil, 0, false + } + i++ + } + if i >= len(sql) || sql[i] != '(' { + return nil, 0, false + } + depth := 0 + start := i + 1 + for ; i < len(sql); i++ { + if !m[i] { + continue + } + switch sql[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + args = append(args, sql[start:i]) + return args, i + 1, true + } + case ',': + if depth == 1 { + args = append(args, sql[start:i]) + start = i + 1 + } + } + } + return nil, 0, false +} + +func ifArgs(args []string) string { + if len(args) != 3 { + return "IF(" + strings.Join(args, ", ") + ")" // leave loud + } + return "CASE WHEN " + strings.TrimSpace(args[0]) + " THEN " + strings.TrimSpace(args[1]) + + " ELSE " + strings.TrimSpace(args[2]) + " END" +} + +func valuesArgs(args []string) string { + if len(args) != 1 { + return "VALUES(" + strings.Join(args, ", ") + ")" + } + return "EXCLUDED." + strings.TrimSpace(args[0]) +} + +func concatArgs(args []string) string { + for i := range args { + args[i] = strings.TrimSpace(args[i]) + } + return "(" + strings.Join(args, " || ") + ")" +} + +func locateArgs(args []string) string { + if len(args) != 2 { + return "LOCATE(" + strings.Join(args, ", ") + ")" // 3-arg form not on bd's path; leave loud + } + return "POSITION(" + strings.TrimSpace(args[0]) + " IN " + strings.TrimSpace(args[1]) + ")" +} + +func instrArgs(args []string) string { + if len(args) != 2 { + return "INSTR(" + strings.Join(args, ", ") + ")" // leave loud + } + // POSITION reverses INSTR's arg order. Placeholder numbering runs later and + // left-to-right, so reordering two args that BOTH carry a `?` would swap their + // $-numbers relative to the caller's bound-arg order β€” a silent wrong-parameter + // mistranslation. bd only ever emits INSTR(col, ?), so guard the ambiguous case + // by leaving it loud to fail at PREPARE rather than swap bindings. + if strings.Contains(args[0], "?") && strings.Contains(args[1], "?") { + return "INSTR(" + strings.Join(args, ", ") + ")" + } + // MySQL INSTR(haystack, needle) β€” note the reversed order vs LOCATE. + return "POSITION(" + strings.TrimSpace(args[1]) + " IN " + strings.TrimSpace(args[0]) + ")" +} + +// mysqlDateFormatISO8601 is the ONLY DATE_FORMAT pattern bd emits (counts.go's +// DepJSONObject renders created_at as RFC3339). Any other format literal is +// left untranslated so it fails loudly at PREPARE rather than silently +// producing a differently-shaped timestamp. +const ( + mysqlDateFormatISO8601 = `'%Y-%m-%dT%H:%i:%sZ'` + pgDateFormatISO8601 = `'YYYY-MM-DD"T"HH24:MI:SS"Z"'` +) + +func dateFormatArgs(args []string) string { + if len(args) != 2 || strings.TrimSpace(args[1]) != mysqlDateFormatISO8601 { + return "DATE_FORMAT(" + strings.Join(args, ", ") + ")" // unknown format: leave loud + } + return "to_char(" + strings.TrimSpace(args[0]) + ", " + pgDateFormatISO8601 + ")" +} + +// jsonObjectArgs maps MySQL's comma-form JSON_OBJECT(k1, v1, k2, v2, …) to +// Postgres jsonb_build_object with the same alternating key/value args. An odd +// (or empty) arg count is not a valid object literal, so it is left loud. +func jsonObjectArgs(args []string) string { + if len(args) == 0 || len(args)%2 != 0 { + return "JSON_OBJECT(" + strings.Join(args, ", ") + ")" // leave loud + } + for i := range args { + args[i] = strings.TrimSpace(args[i]) + } + return "jsonb_build_object(" + strings.Join(args, ", ") + ")" +} + +// rewriteInsertIgnore turns `INSERT IGNORE INTO …` into `INSERT INTO … ON +// CONFLICT DO NOTHING`. bd uses it only for idempotent child-table inserts +// (labels, deps, events, comments) with no trailing clause after VALUES/SELECT. +func rewriteInsertIgnore(sql string) string { + const ins = "insert ignore" // bd emits a single space between the keywords + idx := indexCodeFold(sql, ins) + if idx < 0 { + return sql + } + out := sql[:idx] + "INSERT" + sql[idx+len(ins):] + if !containsCodeFold(out, "ON CONFLICT") { + out = strings.TrimRight(out, " \n\t") + " ON CONFLICT DO NOTHING" + } + return out +} + +// rewriteLabelOrderBy makes label sorting explicit on Postgres. The shared +// query text stays raw (`ORDER BY label` / `ORDER BY issue_id, label`) so the +// Dolt reference path remains unchanged; the SQL backends translate it to a +// code-point collation here. +func rewriteLabelOrderBy(sql string) string { + sql = replaceIdentAll(sql, "ORDER BY issue_id, label", `ORDER BY issue_id, label COLLATE "C"`) + return replaceIdentAll(sql, "ORDER BY label", `ORDER BY label COLLATE "C"`) +} + +// rewriteReplaceInto converts MySQL `REPLACE INTO t (c1, c2, …) VALUES …` into +// `INSERT INTO t (c1, c2, …) VALUES … ON CONFLICT (c1) DO UPDATE SET c2 = +// EXCLUDED.c2, …`. bd only uses REPLACE on key/value tables (config, metadata, +// local_metadata) where the first column is the primary key, so treating column +// 1 as the conflict target and overwriting the rest is faithful. The column +// tokens keep their backticks here; the later rewriteBackticks pass quotes them. +func rewriteReplaceInto(sql string) string { + const rep = "replace into" + idx := indexCodeFold(sql, rep) + if idx < 0 { + return sql + } + out := sql[:idx] + "INSERT INTO" + sql[idx+len(rep):] + paren := strings.Index(out[idx:], "(") + if paren < 0 { + return out // no column list β€” leave loud + } + cols, _, ok := splitCallArgs(out, idx+paren) + if !ok || len(cols) == 0 { + return out + } + target := strings.TrimSpace(cols[0]) + sets := make([]string, 0, len(cols)-1) + for _, c := range cols[1:] { + c = strings.TrimSpace(c) + sets = append(sets, c+" = EXCLUDED."+c) + } + tail := " ON CONFLICT (" + target + ") DO NOTHING" + if len(sets) > 0 { + tail = " ON CONFLICT (" + target + ") DO UPDATE SET " + strings.Join(sets, ", ") + } + return strings.TrimRight(out, " \n\t") + tail +} + +func skipWS(sql string, i int) int { + for i < len(sql) && (sql[i] == ' ' || sql[i] == '\t' || sql[i] == '\n' || sql[i] == '\r') { + i++ + } + return i +} + +// wordAtFold is wordAt with case-insensitive matching (for SQL keywords). +func wordAtFold(sql string, i int, word string) bool { + if i+len(word) > len(sql) || !strings.EqualFold(sql[i:i+len(word)], word) { + return false + } + if i > 0 && isIdentByte(sql[i-1]) { + return false + } + if i+len(word) < len(sql) && isIdentByte(sql[i+len(word)]) { + return false + } + return true +} + +func isIdentWord(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if !isIdentByte(s[i]) { + return false + } + } + return true +} + +// rewriteUpdateSetTargets strips the table-alias qualifier from UPDATE SET +// targets: MySQL accepts `UPDATE t a SET a.col = …`, but Postgres requires the +// SET target to be unqualified (`SET col = …`) β€” it reads `a` as a column +// otherwise. Only the assignment target (left of the top-level `=`) is +// unqualified; alias references on the right-hand side are left intact. +func rewriteUpdateSetTargets(sql string) string { + m := codeMask(sql) + start := skipWS(sql, 0) + if !wordAtFold(sql, start, "update") { + return sql + } + setPos := scanKeyword(sql, m, start, "set") + if setPos < 0 { + return sql + } + clauseStart := setPos + len("set") + clauseEnd := scanKeyword(sql, m, clauseStart, "where") + if clauseEnd < 0 { + if r := scanKeyword(sql, m, clauseStart, "returning"); r >= 0 { + clauseEnd = r + } else { + clauseEnd = len(sql) + } + } + clause := sql[clauseStart:clauseEnd] + cm := codeMask(clause) + var out strings.Builder + depth, segStart := 0, 0 + for j := 0; j < len(clause); j++ { + if cm[j] { + switch clause[j] { + case '(': + depth++ + case ')': + depth-- + case ',': + if depth == 0 { + out.WriteString(unqualifySetTarget(clause[segStart:j])) + out.WriteByte(',') + segStart = j + 1 + } + } + } + } + out.WriteString(unqualifySetTarget(clause[segStart:])) + return sql[:clauseStart] + out.String() + sql[clauseEnd:] +} + +// scanKeyword returns the index of the next depth-0, code-region occurrence of +// keyword (word-boundaried, case-insensitive) at or after from, or -1. +func scanKeyword(sql string, m []bool, from int, keyword string) int { + depth := 0 + for j := from; j < len(sql); j++ { + if !m[j] { + continue + } + switch sql[j] { + case '(': + depth++ + case ')': + depth-- + } + if depth == 0 && wordAtFold(sql, j, keyword) { + return j + } + } + return -1 +} + +func unqualifySetTarget(seg string) string { + cm := codeMask(seg) + depth, eq := 0, -1 + for j := 0; j < len(seg) && eq < 0; j++ { + if !cm[j] { + continue + } + switch seg[j] { + case '(': + depth++ + case ')': + depth-- + case '=': + if depth == 0 { + eq = j + } + } + } + if eq < 0 { + return seg + } + lhs := seg[:eq] + trimmed := strings.TrimSpace(lhs) + dot := strings.IndexByte(trimmed, '.') + if dot <= 0 || !isIdentWord(trimmed[:dot]) || !isIdentWord(trimmed[dot+1:]) { + return seg + } + lead := lhs[:len(lhs)-len(strings.TrimLeft(lhs, " \t\r\n"))] + return lead + trimmed[dot+1:] + " " + seg[eq:] +} + +// onConflictTargets maps a table to the columns Postgres needs as the ON +// CONFLICT arbiter (the primary/unique key MySQL's ON DUPLICATE KEY infers). +// Only tables that use a DO UPDATE upsert need an entry; DO NOTHING needs none. +var onConflictTargets = map[string]string{ + "issues": "id", + "wisps": "id", + "issue_counter": "prefix", + "child_counters": "parent_id", + "wisp_child_counters": "parent_id", + "repo_mtimes": "repo_path", + "federation_peers": "name", +} + +// rewriteOnDuplicateKey converts `… ON DUPLICATE KEY UPDATE ` into +// Postgres `… ON CONFLICT () DO UPDATE SET `, or `ON CONFLICT DO +// NOTHING` when the body is an idempotent self-assignment (`col = col`). The ON +// DUPLICATE clause is always last in the statement, so the body runs to the end. +func rewriteOnDuplicateKey(sql string) string { + const odk = "on duplicate key update" + i := indexCodeFold(sql, odk) + if i < 0 { + return sql + } + head := strings.TrimRight(sql[:i], " \n\t") + body := strings.TrimSpace(sql[i+len(odk):]) + if isSelfAssign(body) { + return head + " ON CONFLICT DO NOTHING" + } + // Inside the upsert body, VALUES(col) means the proposed row value + // (Postgres EXCLUDED.col) and IF(c,a,b) is the ternary (CASE). Translate them + // here, where they can't be confused with the INSERT ... VALUES keyword. + body = rewriteFuncCalls(body, "IF", ifArgs) + body = rewriteFuncCalls(body, "VALUES", valuesArgs) + target := onConflictTargets[insertTable(sql)] + if target == "" { + // Unknown upsert target: emit a form that fails PREPARE loudly rather + // than guessing an arbiter (surfaced by the corpus gate). + return head + " ON CONFLICT () DO UPDATE SET " + body + } + return head + " ON CONFLICT (" + target + ") DO UPDATE SET " + body +} + +// isSelfAssign reports whether body is a single `x = x` assignment. +func isSelfAssign(body string) bool { + parts := strings.SplitN(body, "=", 2) + if len(parts) != 2 { + return false + } + l := strings.TrimSpace(parts[0]) + r := strings.TrimSpace(strings.TrimRight(parts[1], " \n\t")) + return l != "" && l == r && !strings.ContainsAny(l, " (),") +} + +// insertTable extracts the target table from a leading `INSERT [IGNORE] INTO +//
`. The table is concrete at runtime (fmt.Sprintf filled it in). +func insertTable(sql string) string { + toks := strings.Fields(sql) + for i := 0; i+1 < len(toks); i++ { + if strings.EqualFold(toks[i], "INTO") { + t := strings.Trim(toks[i+1], "`\"") + if p := strings.IndexAny(t, "("); p >= 0 { + t = t[:p] + } + return t + } + } + return "" +} + +// rewriteLiteralJSONUnquote handles the fixed-path form +// JSON_UNQUOTE(JSON_EXTRACT(, '$.')) (the waits-for gate check), +// rewriting it to ( #>> '{}'). Only the single-level literal-path +// variant is handled; dynamic ?-path metadata filters translate separately. +func rewriteLiteralJSONUnquote(sql string) string { + const marker = "json_unquote(json_extract(" + for { + i := indexCodeFold(sql, marker) + if i < 0 { + return sql + } + open := i + len("JSON_UNQUOTE(") + len("JSON_EXTRACT") + args, extractEnd, ok := splitCallArgs(sql, open) + if !ok || len(args) != 2 || extractEnd >= len(sql) || sql[extractEnd] != ')' { + return sql // malformed / not the literal form: leave loud + } + key := jsonPathToKey(strings.TrimSpace(args[1])) + if key == "" { + return sql // dynamic path: leave for the param translator + } + repl := "(" + strings.TrimSpace(args[0]) + " #>> '{" + key + "}')" + sql = sql[:i] + repl + sql[extractEnd+1:] + } +} + +// rewriteDynamicJSONPath translates the dynamic ?-path metadata filters +// sqlbuild emits (filter.go): the has-key form JSON_EXTRACT(metadata, ?) IS NOT +// NULL and the equality form JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) = ?. The +// MySQL path argument (e.g. '$.sprint' or '$."gc.routed_to"') is a bound +// parameter, so the bare-key conversion happens at runtime via the +// bd_mysql_jsonkey SQL function (defined in the Postgres schema): +// +// JSON_UNQUOTE(JSON_EXTRACT(, ?)) -> ( ->> bd_mysql_jsonkey(?)) +// JSON_EXTRACT(, ?) -> ( -> bd_mysql_jsonkey(?)) +// +// The unquote form is rewritten first so its inner JSON_EXTRACT is consumed +// before the bare-extract pass runs. Literal-path forms (a quoted '$.key' +// argument) are left untouched here; rewriteLiteralJSONUnquote already handled +// them and any bare literal JSON_EXTRACT is left loud. +func rewriteDynamicJSONPath(sql string) string { + sql = rewriteDynamicJSONUnquote(sql) + sql = rewriteDynamicJSONExtract(sql) + return sql +} + +func rewriteDynamicJSONUnquote(sql string) string { + const marker = "json_unquote(json_extract(" + searchFrom := 0 + for { + i := indexCodeFoldFrom(sql, marker, searchFrom) + if i < 0 { + return sql + } + open := i + len("JSON_UNQUOTE(") + len("JSON_EXTRACT") + args, extractEnd, ok := splitCallArgs(sql, open) + if !ok || len(args) != 2 || extractEnd >= len(sql) || sql[extractEnd] != ')' { + searchFrom = i + len(marker) // malformed / not this form: skip + continue + } + if strings.TrimSpace(args[1]) != "?" { + searchFrom = i + len(marker) // literal path: leave for the literal pass + continue + } + repl := "(" + strings.TrimSpace(args[0]) + " ->> bd_mysql_jsonkey(?))" + sql = sql[:i] + repl + sql[extractEnd+1:] // +1 consumes the outer JSON_UNQUOTE ')' + searchFrom = i + len(repl) + } +} + +func rewriteDynamicJSONExtract(sql string) string { + searchFrom := 0 + for { + idx := findCallStart(sql, "json_extract", searchFrom) + if idx < 0 { + return sql + } + args, end, ok := splitCallArgs(sql, idx+len("JSON_EXTRACT")) + if !ok || len(args) != 2 || strings.TrimSpace(args[1]) != "?" { + searchFrom = idx + len("JSON_EXTRACT") // literal / malformed path: leave loud + continue + } + repl := "(" + strings.TrimSpace(args[0]) + " -> bd_mysql_jsonkey(?))" + sql = sql[:idx] + repl + sql[end:] + searchFrom = idx + len(repl) + } +} + +// rewriteCastChar rewrites CAST( AS CHAR) and CAST( AS CHAR(n)) into +// ()::text. bd emits CAST(metadata AS CHAR) to serialize a JSON column to +// its text form (counts.go DepJSONObject). This rewrite is MANDATORY, not +// cosmetic: Postgres reads a bare CHAR as char(1), so CAST(jsonb AS CHAR) +// prepares silently and truncates the value to a single character. +func rewriteCastChar(sql string) string { + searchFrom := 0 + for { + idx := findCallStart(sql, "cast", searchFrom) + if idx < 0 { + return sql + } + args, end, ok := splitCallArgs(sql, idx+len("CAST")) + if !ok || len(args) != 1 { + searchFrom = idx + len("CAST") + continue + } + expr, isChar := splitCastAsChar(args[0]) + if !isChar { + searchFrom = idx + len("CAST") // not AS CHAR (e.g. AS SIGNED): leave loud + continue + } + repl := "(" + strings.TrimSpace(expr) + ")::text" + sql = sql[:idx] + repl + sql[end:] + searchFrom = idx + len(repl) + } +} + +// splitCastAsChar reports whether inner has the shape " AS CHAR" or +// " AS CHAR(n)" and, if so, returns . The AS is matched at paren +// depth 0 in a code region, so an AS nested inside (e.g. a nested CAST) +// is ignored. +func splitCastAsChar(inner string) (expr string, ok bool) { + m := codeMask(inner) + depth := 0 + for i := 0; i < len(inner); i++ { + if !m[i] { + continue + } + switch inner[i] { + case '(': + depth++ + case ')': + depth-- + } + if depth == 0 && wordAtFold(inner, i, "as") { + if isCharType(strings.TrimSpace(inner[i+len("as"):])) { + return inner[:i], true + } + return "", false + } + } + return "", false +} + +// isCharType reports whether tail is a CHAR type token: "CHAR" or "CHAR(n)". +func isCharType(tail string) bool { + if !wordAtFold(tail, 0, "char") { + return false + } + rest := strings.TrimSpace(tail[len("char"):]) + if rest == "" { + return true // CHAR + } + return rest[0] == '(' && rest[len(rest)-1] == ')' // CHAR(n) +} + +// jsonPathToKey converts a simple literal MySQL JSON path '$.key' into the bare +// key, or "" if it is not a simple single-level literal. +func jsonPathToKey(lit string) string { + if len(lit) < 4 || lit[0] != '\'' || lit[len(lit)-1] != '\'' { + return "" + } + p := lit[1 : len(lit)-1] + if !strings.HasPrefix(p, "$.") { + return "" + } + k := strings.Trim(p[2:], `"`) + if k == "" || strings.ContainsAny(k, ".[]*") { + return "" + } + return k +} + +func indexCodeFold(sql, sub string) int { return indexCodeFoldFrom(sql, sub, 0) } + +// indexCodeFoldFrom is indexCodeFold starting the scan at `from`. Rewrites that +// may legitimately leave a match in place (a literal path they must skip) use +// this to advance past it instead of looping forever. +func indexCodeFoldFrom(sql, sub string, from int) int { + if from < 0 { + from = 0 + } + m := codeMask(sql) + lower := strings.ToLower(sub) + for i := from; i+len(sub) <= len(sql); i++ { + if m[i] && strings.ToLower(sql[i:i+len(sub)]) == lower { + return i + } + } + return -1 +} + +func containsCodeFold(sql, sub string) bool { return indexCodeFold(sql, sub) >= 0 } diff --git a/internal/storage/pgdialect/translate_test.go b/internal/storage/pgdialect/translate_test.go new file mode 100644 index 000000000..8c2799bf1 --- /dev/null +++ b/internal/storage/pgdialect/translate_test.go @@ -0,0 +1,98 @@ +package pgdialect + +import "testing" + +func TestTranslate(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + { + name: "placeholders numbered", + in: `SELECT * FROM issues WHERE status = ? AND priority = ?`, + want: `SELECT * FROM issues WHERE status = $1 AND priority = $2`, + }, + { + name: "placeholder inside string is left alone", + in: `SELECT '?', assignee FROM issues WHERE id = ?`, + want: `SELECT '?', assignee FROM issues WHERE id = $1`, + }, + { + name: "backtick reserved-word identifier", + in: "SELECT value FROM config WHERE `key` = ?", + want: `SELECT value FROM config WHERE "key" = $1`, + }, + { + name: "INSERT IGNORE -> ON CONFLICT DO NOTHING", + in: `INSERT IGNORE INTO labels (issue_id, label) VALUES (?, ?)`, + want: `INSERT INTO labels (issue_id, label) VALUES ($1, $2) ON CONFLICT DO NOTHING`, + }, + { + name: "ON DUPLICATE KEY type=type -> ON CONFLICT DO NOTHING", + in: `INSERT INTO dependencies (id, issue_id, depends_on_issue_id, type, created_by, created_at) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE type = type`, + want: `INSERT INTO dependencies (id, issue_id, depends_on_issue_id, type, created_by, created_at) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING`, + }, + { + name: "issue upsert VALUES(col) -> EXCLUDED.col with (id) target", + in: `INSERT INTO issues (id, title) VALUES (?, ?) ON DUPLICATE KEY UPDATE title = VALUES(title)`, + want: `INSERT INTO issues (id, title) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET title = EXCLUDED.title`, + }, + { + name: "stale-guard IF(VALUES...)-> CASE WHEN EXCLUDED", + in: `INSERT INTO issues (id, title, updated_at) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE title = IF(VALUES(updated_at) > updated_at, VALUES(title), title)`, + want: `INSERT INTO issues (id, title, updated_at) VALUES ($1, $2, $3) ON CONFLICT (id) DO UPDATE SET title = CASE WHEN EXCLUDED.updated_at > updated_at THEN EXCLUDED.title ELSE title END`, + }, + { + name: "child counter GREATEST upsert with (parent_id) target", + in: `INSERT INTO child_counters (parent_id, last_child) VALUES (?, ?) ON DUPLICATE KEY UPDATE last_child = GREATEST(last_child, ?)`, + want: `INSERT INTO child_counters (parent_id, last_child) VALUES ($1, $2) ON CONFLICT (parent_id) DO UPDATE SET last_child = GREATEST(last_child, $3)`, + }, + { + name: "UPDATE SET unqualifies target, keeps RHS alias reference", + in: `UPDATE issues i SET i.is_blocked = 1, i.updated_at = i.updated_at WHERE i.id = ?`, + want: `UPDATE issues i SET is_blocked = 1, updated_at = i.updated_at WHERE i.id = $1`, + }, + { + name: "REPLACE INTO -> INSERT ON CONFLICT DO UPDATE", + in: "REPLACE INTO config (`key`, value) VALUES (?, ?)", + want: `INSERT INTO config ("key", value) VALUES ($1, $2) ON CONFLICT ("key") DO UPDATE SET value = EXCLUDED.value`, + }, + { + name: "CONCAT -> || (with LIKE and escaped percent)", + in: `SELECT id FROM issues WHERE id LIKE CONCAT(?, '.%')`, + want: `SELECT id FROM issues WHERE id LIKE ($1 || '.%')`, + }, + { + name: "LOCATE(CONCAT(...)) nested -> POSITION(... IN ...)", + in: `SELECT 1 WHERE LOCATE(CONCAT(',', e.issue_id, ','), d.path) = 0`, + want: `SELECT 1 WHERE POSITION((',' || e.issue_id || ',') IN d.path) = 0`, + }, + { + name: "UTC_TIMESTAMP()", + in: `SELECT 1 WHERE defer_until IS NULL OR defer_until <= UTC_TIMESTAMP()`, + want: `SELECT 1 WHERE defer_until IS NULL OR defer_until <= (CURRENT_TIMESTAMP AT TIME ZONE 'utc')`, + }, + { + name: "literal JSON gate path", + in: `SELECT 1 WHERE JSON_UNQUOTE(JSON_EXTRACT(d.metadata, '$.gate')) = 'any-children'`, + want: `SELECT 1 WHERE (d.metadata #>> '{gate}') = 'any-children'`, + }, + { + name: "binary label collation", + in: `SELECT label FROM labels WHERE issue_id = ? ORDER BY label`, + want: `SELECT label FROM labels WHERE issue_id = $1 ORDER BY label COLLATE "C"`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := Translate(tc.in) + if err != nil { + t.Fatalf("Translate error: %v", err) + } + if got != tc.want { + t.Errorf("Translate mismatch\n in: %s\n got: %s\n want: %s", tc.in, got, tc.want) + } + }) + } +} diff --git a/internal/storage/postgres/collation.go b/internal/storage/postgres/collation.go new file mode 100644 index 000000000..d3f8ee205 --- /dev/null +++ b/internal/storage/postgres/collation.go @@ -0,0 +1,49 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "os" + "strings" +) + +// collationParityOK reports whether a Postgres database-level collation orders text +// by byte / Unicode code point β€” the ordering the shared SQL layer needs to match +// Dolt's utf8mb4_0900_bin oracle. C and POSIX are exact code-point order, and the +// C.* family (e.g. C.UTF-8) also sorts by code point. glibc/ICU locales such as +// en_US.UTF-8 sort linguistically and silently diverge from Dolt. +func collationParityOK(collation string) bool { + c := strings.ToUpper(strings.TrimSpace(collation)) + return c == "C" || c == "POSIX" || strings.HasPrefix(c, "C.") +} + +// databaseCollation returns the connected database's lc_collate. +func databaseCollation(ctx context.Context, db *sql.DB) (string, error) { + var collation string + if err := db.QueryRowContext(ctx, "SHOW lc_collate").Scan(&collation); err != nil { + return "", err + } + return collation, nil +} + +// warnOnCollationDivergence prints a prominent stderr warning when the connected +// database is not code-point ordered. Text columns use the database default +// collation β€” per-column COLLATE "C" was dropped because it breaks the shared +// recursive CTEs (see schema.go) β€” so a non-C/POSIX database makes every text +// ORDER BY the shared layer emits (bd list, bd ready, label ordering) sort +// linguistically and diverge from Dolt with no error and no other signal. +// +// It is best-effort: a query failure is swallowed so the check can never block +// opening the store. Running from Provision means `bd init --backend=postgres` +// surfaces the requirement at setup and every later open re-warns until the +// operator recreates the database with a code-point collation. This mirrors the +// Dolt backend's open-time stderr warnings. +func warnOnCollationDivergence(ctx context.Context, db *sql.DB) { + collation, err := databaseCollation(ctx, db) + if err != nil || collationParityOK(collation) { + return + } + fmt.Fprintf(os.Stderr, "Warning: Postgres database collation is %q, not C/POSIX. Text ORDER BY (bd list, bd ready, labels) sorts linguistically and diverges from Dolt's code-point order.\n", collation) + fmt.Fprintln(os.Stderr, " Recreate the database with a code-point collation, e.g. CREATE DATABASE ... TEMPLATE template0 LC_COLLATE 'C' LC_CTYPE 'C';") +} diff --git a/internal/storage/postgres/collation_test.go b/internal/storage/postgres/collation_test.go new file mode 100644 index 000000000..472100fae --- /dev/null +++ b/internal/storage/postgres/collation_test.go @@ -0,0 +1,29 @@ +package postgres + +import "testing" + +func TestCollationParityOK(t *testing.T) { + cases := []struct { + collation string + want bool + }{ + {"C", true}, + {"POSIX", true}, + {"c", true}, // case-insensitive + {"posix", true}, // case-insensitive + {" C ", true}, // trimmed + {"C.UTF-8", true}, // C.* family sorts by code point + {"C.utf8", true}, + {"en_US.UTF-8", false}, + {"en_US.utf8", false}, + {"en_GB.UTF-8", false}, + {"und-x-icu", false}, // ICU root + {"de_DE.UTF-8", false}, + {"", false}, + } + for _, tc := range cases { + if got := collationParityOK(tc.collation); got != tc.want { + t.Errorf("collationParityOK(%q) = %v, want %v", tc.collation, got, tc.want) + } + } +} diff --git a/internal/storage/postgres/completeness_test.go b/internal/storage/postgres/completeness_test.go new file mode 100644 index 000000000..e95bc0182 --- /dev/null +++ b/internal/storage/postgres/completeness_test.go @@ -0,0 +1,107 @@ +package postgres + +import ( + "os" + "regexp" + "sort" + "testing" + + "github.com/steveyegge/beads/internal/storage/conformance" +) + +// legitimatelyUnsupported is the EXPLICIT denominator of every storage.DoltStorage +// method the Postgres wedge deliberately does not implement, each with a reason. +// +// The interface-completeness gate (TestInterfaceCompleteness) asserts the +// generated capability shell (unsupported_gen.go) contains EXACTLY these methods +// β€” no more, no less. This is the guard the red-team had to stand in for: a +// method that silently resolves to the typed-unsupported shell (the class that +// bit us: DeleteIssues, GetCustomStatuses, the batch reads, …) fails this test +// at build time instead of failing a user at runtime. Adding a method to the +// shell without an entry here is a hard error that forces the triage question: +// "implement it in sqlkit, or record here why the wedge legitimately omits it." +var legitimatelyUnsupported = map[string]string{ + // VersionControl β€” the Dolt commit graph. The wedge's whole premise is + // "bd on Postgres, history/version-control simply absent". + "Branch": "VC: Dolt branch", "Checkout": "VC: Dolt checkout", + "CurrentBranch": "VC", "DeleteBranch": "VC", "ListBranches": "VC", + // Commit/CommitWithConfig/CommitMergeResolution are NO-OPS on Postgres (data + // is already durable per-tx), not unsupported β€” implemented on postgres.Store. + "CommitExists": "VC", "GetCurrentCommit": "VC: Dolt hash", "Status": "VC", + "Log": "VC: dolt_log", "Merge": "VC", "GetConflicts": "VC", "ResolveConflicts": "VC", + + // HistoryViewer β€” Dolt time-travel. + "History": "history: dolt_log", "AsOf": "history: as-of ref", "Diff": "history: dolt_diff", + + // RemoteStore β€” Dolt remotes. + "AddRemote": "remote", "RemoveRemote": "remote", "HasRemote": "remote", "ListRemotes": "remote", + "Push": "remote", "Pull": "remote", "ForcePush": "remote", "PushRemote": "remote", + "PullRemote": "remote", "Fetch": "remote", "PushTo": "remote", "PullFrom": "remote", + + // SyncStore β€” Dolt peer sync. + "Sync": "sync", "SyncStatus": "sync", + + // FederationStore β€” Dolt-remote-backed peer table. + "AddFederationPeer": "federation", "GetFederationPeer": "federation", + "ListFederationPeers": "federation", "RemoveFederationPeer": "federation", + + // CompactionStore β€” bound to Dolt commit hashes. + "CheckEligibility": "compaction", "ApplyCompaction": "compaction", + "GetTier1Candidates": "compaction", "GetTier2Candidates": "compaction", + "SnapshotIssue": "compaction", "GetCompactionSnapshot": "compaction", "RestoreFromSnapshot": "compaction", + + // MergeSlot β€” the Dolt merge-serialization primitive. + "MergeSlotCreate": "merge-slot", "MergeSlotCheck": "merge-slot", + "MergeSlotAcquire": "merge-slot", "MergeSlotRelease": "merge-slot", +} + +var shellMethodRe = regexp.MustCompile(`func \(unsupportedDoltStorage\) ([A-Za-z0-9]+)\(`) + +// TestInterfaceCompleteness is the interface-completeness gate: the generated +// shell must equal the audited legitimatelyUnsupported set, both directions. +func TestInterfaceCompleteness(t *testing.T) { + src, err := os.ReadFile("unsupported_gen.go") + if err != nil { + t.Fatalf("read unsupported_gen.go: %v", err) + } + shell := map[string]bool{} + for _, m := range shellMethodRe.FindAllStringSubmatch(string(src), -1) { + shell[m[1]] = true + } + if len(shell) == 0 { + t.Fatal("parsed 0 shell methods β€” regex or file drift") + } + + // (1) No SILENT gap: every shell method must be explicitly justified. + var unjustified []string + for m := range shell { + if _, ok := legitimatelyUnsupported[m]; !ok { + unjustified = append(unjustified, m) + } + } + sort.Strings(unjustified) + for _, m := range unjustified { + t.Errorf("method %q resolves to the typed-unsupported shell but is NOT in legitimatelyUnsupported: implement it on *sqlkit.Store, or add it here with a reason", m) + } + + // (2) No STALE allowlist: every justified method must still be in the shell + // (else it was implemented and the entry should be removed). + var stale []string + for m := range legitimatelyUnsupported { + if !shell[m] { + stale = append(stale, m) + } + } + sort.Strings(stale) + for _, m := range stale { + t.Errorf("method %q is in legitimatelyUnsupported but no longer in the shell (implemented?): remove the allowlist entry", m) + } +} + +// TestUnsupportedContract is the behavioral complement to TestInterfaceCompleteness: +// every allowlisted method must actually return a typed storage.ErrUnsupported when +// called, not panic or return a different error. DB-free β€” the generated stubs ignore +// their receiver, so a zero-value store answers them. +func TestUnsupportedContract(t *testing.T) { + conformance.RunUnsupportedContract(t, &Store{}, legitimatelyUnsupported) +} diff --git a/internal/storage/postgres/conformance_test.go b/internal/storage/postgres/conformance_test.go new file mode 100644 index 000000000..d7eb19446 --- /dev/null +++ b/internal/storage/postgres/conformance_test.go @@ -0,0 +1,89 @@ +package postgres + +import ( + "context" + "fmt" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/conformance" + "github.com/steveyegge/beads/internal/storage/pgdialect" +) + +// TestConformance runs bd's backend-agnostic storage conformance suite +// (internal/storage/conformance, ~40 encoded behavior tests) against the +// Postgres wedge. This is the real conformance measurement β€” far broader than +// the smoke. Every failure is a PG gap: either an allowlisted-unsupported method +// (expected β€” see completeness_test.go) or a latent divergence nothing else +// caught (the interesting ones). Gated on BEADS_PG_TEST_URL. +func TestConformance(t *testing.T) { + url := os.Getenv("BEADS_PG_TEST_URL") + if url == "" { + t.Skip("BEADS_PG_TEST_URL not set") + } + conformance.RunAll(t, pgConformanceFactory(url)) +} + +// TestDeferredReads runs the shared deferred non-version-control reads (statistics, +// external-ref, stale) that the wedge now implements through issueops, as a GREEN +// gate. RunAll stays the fail-loud measurement (red on genuinely Dolt-only methods +// like slots), so this focused gate is what conformance.sh runs for the wedge. +// Gated on BEADS_PG_TEST_URL. +func TestDeferredReads(t *testing.T) { + url := os.Getenv("BEADS_PG_TEST_URL") + if url == "" { + t.Skip("BEADS_PG_TEST_URL not set") + } + conformance.RunDeferredReads(t, pgConformanceFactory(url)) +} + +// TestPortableMethods runs the full portable-method behavior contract (molecule/ +// repo-mtime/streams/counts/comment/rekey/batch). Red until the methods are wired into +// sqlkit; green after. Gated on BEADS_PG_TEST_URL. +func TestPortableMethods(t *testing.T) { + url := os.Getenv("BEADS_PG_TEST_URL") + if url == "" { + t.Skip("BEADS_PG_TEST_URL not set") + } + conformance.RunPortableMethods(t, pgConformanceFactory(url)) +} + +// pgConformanceFactory provisions a fresh, isolated schema per sub-test against the +// Postgres server at url, seeded with issue_prefix as `bd init` leaves it. +func pgConformanceFactory(url string) conformance.Factory { + base := time.Now().UnixNano() + var seq int64 + return func(t *testing.T) storage.DoltStorage { + ctx := context.Background() + schema := fmt.Sprintf("conf_%d_%d", base, atomic.AddInt64(&seq, 1)) + + raw, err := pgdialect.OpenRaw(url, schema) + if err != nil { + t.Fatalf("OpenRaw: %v", err) + } + if err := InitSchema(ctx, raw, schema); err != nil { + _ = raw.Close() + t.Fatalf("InitSchema: %v", err) + } + _ = raw.Close() + + st, err := New(ctx, Config{DSN: url, Schema: schema}) + if err != nil { + t.Fatalf("New: %v", err) + } + // The conformance factory contract wants an init'd store (issue_prefix + // set), as `bd init` leaves it. No Commit β€” PG has no commit graph. + if err := st.SetConfig(ctx, "issue_prefix", "test"); err != nil { + t.Fatalf("SetConfig(issue_prefix): %v", err) + } + t.Cleanup(func() { + _, _ = st.DB().ExecContext(context.Background(), + fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) + _ = st.Close() + }) + return st + } +} diff --git a/internal/storage/postgres/credential.go b/internal/storage/postgres/credential.go new file mode 100644 index 000000000..f5eb12cf9 --- /dev/null +++ b/internal/storage/postgres/credential.go @@ -0,0 +1,67 @@ +package postgres + +import ( + "context" + "fmt" + "os" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/creds" + "github.com/steveyegge/beads/internal/storage/pgdialect" +) + +// resolveDSNCredential returns baseDSN with a credential resolved and placed, ready to +// connect. The precedence, high to low: +// +// - BEADS_PG_CREDENTIAL_COMMAND set is a hard error: an identity credential is +// presented as a username to a gateway, and this backend connects directly; +// - the DSN already carries a password (a full BEADS_POSTGRES_URL, or an explicit +// --pg-url) β€” it wins outright, no ladder is applied; +// - BEADS_PG_PASSWORD_COMMAND β€” a credential command (e.g. Vault / RDS-IAM / GCP-IAM); +// run at open time, cached until near expiry; +// - BEADS_PG_PASSWORD β€” a static password; +// - the credentials file [host:port] ($BEADS_CREDENTIALS_FILE or +// ~/.config/beads/credentials); +// - nothing configured β€” baseDSN is returned untouched so pgx's own libpq +// fallbacks (PGPASSWORD, ~/.pgpass, PGPASSFILE) still apply. +// +// It fails closed: if a configured credential command errors, the open aborts β€” it +// never silently downgrades to a lower rung. +func resolveDSNCredential(ctx context.Context, cfg *configfile.Config, baseDSN string) (string, error) { + if os.Getenv("BEADS_PG_CREDENTIAL_COMMAND") != "" { + return "", fmt.Errorf("postgres: BEADS_PG_CREDENTIAL_COMMAND is reserved: an identity credential is presented as a username to a gateway, and the postgres backend connects directly to the database with no gateway to present it to; use BEADS_PG_PASSWORD_COMMAND for a helper that produces a password") + } + // host:port keys the credentials-file lookup; ok=false (unix socket / unparseable) + // leaves the file rung not-configured (zero Host/Port), never failing. + host, port, _ := pgdialect.HostPort(baseDSN) + return resolveDSNWithSources(ctx, baseDSN, + creds.CommandSource{Command: cfg.GetPostgresPasswordCommand(), Kind: creds.KindSecret, Label: "BEADS_PG_PASSWORD_COMMAND"}, + creds.EnvSource{Var: "BEADS_PG_PASSWORD"}, + creds.FileSource{Host: host, Port: port, Lookup: configfile.LookupCredentialsPassword, Label: "credentials-file"}, + ) +} + +// resolveDSNWithSources is the testable core of resolveDSNCredential: it applies the +// given credential ladder to baseDSN. A DSN that already carries a password wins +// outright; otherwise the first configured source supplies the credential, and +// nothing-configured leaves the DSN untouched for pgx's libpq fallbacks. +func resolveDSNWithSources(ctx context.Context, baseDSN string, sources ...creds.Source) (string, error) { + if pgdialect.HasPassword(baseDSN) { + return baseDSN, nil + } + cred, ok, err := creds.ResolveLadder(ctx, sources...) + if err != nil { + return "", err + } + if !ok { + return baseDSN, nil + } + // The SQL backends take a secret in the password slot; an identity credential + // (presented as a username to a gateway) has no home on a direct connection, so + // refuse it loudly rather than land a token where a password belongs. Defense in + // depth behind the BEADS_PG_CREDENTIAL_COMMAND reject above. + if cred.Kind != creds.KindSecret { + return "", fmt.Errorf("postgres: credential from %s is an identity, not a password; the postgres backend connects directly and has no gateway to present it to", cred.Source) + } + return pgdialect.WithCredential(baseDSN, cred.Username, cred.Value) +} diff --git a/internal/storage/postgres/credential_test.go b/internal/storage/postgres/credential_test.go new file mode 100644 index 000000000..b2c87572b --- /dev/null +++ b/internal/storage/postgres/credential_test.go @@ -0,0 +1,260 @@ +package postgres + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/creds" +) + +const baseDSN = "postgres://bts@127.0.0.1:5432/db" + +// TestMain neutralizes every ambient credential source the rungs could read, so tests +// are hermetic: the beads credentials file, plus pgx's own PGPASSWORD/~/.pgpass (which +// pgx.ParseConfig folds into a connection at parse/connect time). Tests that exercise +// the file rung override BEADS_CREDENTIALS_FILE per-test with t.Setenv. +func TestMain(m *testing.M) { + dir, err := os.MkdirTemp("", "beads-pg-cred-test") + if err != nil { + panic(err) + } + os.Setenv("BEADS_CREDENTIALS_FILE", filepath.Join(dir, "none")) + os.Unsetenv("PGPASSWORD") + os.Setenv("PGPASSFILE", filepath.Join(dir, "no-pgpass")) + code := m.Run() + _ = os.RemoveAll(dir) + os.Exit(code) +} + +// password parses out and returns the password pgx sees, failing the test on a parse error. +func password(t *testing.T, dsn string) string { + t.Helper() + cfg, err := pgx.ParseConfig(dsn) + if err != nil { + t.Fatalf("pgx.ParseConfig(%q): %v", dsn, err) + } + return cfg.Password +} + +// user parses out and returns the username pgx sees. +func user(t *testing.T, dsn string) string { + t.Helper() + cfg, err := pgx.ParseConfig(dsn) + if err != nil { + t.Fatalf("pgx.ParseConfig(%q): %v", dsn, err) + } + return cfg.User +} + +// writeCredsFile writes an INI credentials file and returns its path. +func writeCredsFile(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "credentials") + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func TestResolveDSNCredentialCommand(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD_COMMAND", "printf cmd-pw") + t.Setenv("BEADS_PG_PASSWORD", "") + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "cmd-pw" { + t.Fatalf("password = %q, want cmd-pw", pw) + } +} + +func TestResolveDSNCredentialEnv(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD_COMMAND", "") + t.Setenv("BEADS_PG_PASSWORD", "env-pw") + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "env-pw" { + t.Fatalf("password = %q, want env-pw", pw) + } +} + +// The command out-ranks the static env password. +func TestResolveDSNCredentialCommandBeatsEnv(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD_COMMAND", "printf cmd-wins") + t.Setenv("BEADS_PG_PASSWORD", "env-loses") + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "cmd-wins" { + t.Fatalf("password = %q, want cmd-wins", pw) + } +} + +// A failing command aborts the open and does NOT fall through to the env password. +func TestResolveDSNCredentialFailsClosed(t *testing.T) { + // A bare token with whitespace is rejected by parseCredential β€” a configured error. + t.Setenv("BEADS_PG_PASSWORD_COMMAND", `printf 'access denied'`) + t.Setenv("BEADS_PG_PASSWORD", "env-would-be-wrong") + _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err == nil { + t.Fatal("expected an error (fail-closed); a broken command must not fall through to BEADS_PG_PASSWORD") + } +} + +// Nothing configured: the base DSN comes back untouched (pgx's own PGPASSWORD/.pgpass +// fallbacks then apply at connect). +func TestResolveDSNCredentialNothingConfigured(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD_COMMAND", "") + t.Setenv("BEADS_PG_PASSWORD", "") + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if got != baseDSN { + t.Fatalf("dsn = %q, want it unchanged (%q)", got, baseDSN) + } +} + +// A DSN that already carries a password wins outright β€” the ladder is not applied. +func TestResolveDSNCredentialExistingPasswordWins(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD_COMMAND", "printf should-not-run") + t.Setenv("BEADS_PG_PASSWORD", "should-not-run") + withPw := "postgres://bts:already-here@127.0.0.1:5432/db" + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, withPw) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "already-here" { + t.Fatalf("password = %q, want already-here (existing password must win)", pw) + } +} + +// The credentials file resolves the password when no command or env is set. +func TestResolveDSNCredentialFile(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD_COMMAND", "") + t.Setenv("BEADS_PG_PASSWORD", "") + t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:5432]\npassword=file-pw\n")) + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "file-pw" { + t.Fatalf("password = %q, want file-pw", pw) + } +} + +// The env password out-ranks the credentials file. +func TestResolveDSNCredentialEnvBeatsFile(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD_COMMAND", "") + t.Setenv("BEADS_PG_PASSWORD", "env-pw") + t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:5432]\npassword=file-pw\n")) + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if pw := password(t, got); pw != "env-pw" { + t.Fatalf("password = %q, want env-pw (env out-ranks file)", pw) + } +} + +// A file section for a different endpoint does not match: the DSN is untouched. +func TestResolveDSNCredentialFileWrongSectionMisses(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD_COMMAND", "") + t.Setenv("BEADS_PG_PASSWORD", "") + t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:9999]\npassword=file-pw\n")) + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if got != baseDSN { + t.Fatalf("dsn = %q, want it unchanged (wrong section must not match)", got) + } +} + +// A broken command fails closed even when the file could supply a valid password. +func TestResolveDSNCredentialBrokenCommandNotRescuedByFile(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD_COMMAND", "false") + t.Setenv("BEADS_PG_PASSWORD", "") + t.Setenv("BEADS_CREDENTIALS_FILE", writeCredsFile(t, "[127.0.0.1:5432]\npassword=file-pw\n")) + if _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN); err == nil { + t.Fatal("expected fail-closed; the credentials file must not rescue a broken command") + } +} + +// A set BEADS_PG_CREDENTIAL_COMMAND is a hard error, pre-empting even a +// password-bearing DSN and any lower rung. +func TestResolveDSNCredentialRejectsReservedIdentityVar(t *testing.T) { + t.Setenv("BEADS_PG_CREDENTIAL_COMMAND", "get-credential") + t.Setenv("BEADS_PG_PASSWORD", "would-otherwise-work") + _, err := resolveDSNCredential(context.Background(), &configfile.Config{}, "postgres://bts:has-pw@127.0.0.1:5432/db") + if err == nil || !strings.Contains(err.Error(), "BEADS_PG_CREDENTIAL_COMMAND") { + t.Fatalf("expected a reserved-var rejection naming BEADS_PG_CREDENTIAL_COMMAND, got %v", err) + } +} + +// staticSource is a test Source with a fixed credential, used to exercise placement +// and the identity-refusal guard directly. +type staticSource struct{ cred creds.Credential } + +func (s staticSource) Name() string { return "test" } +func (s staticSource) Resolve(context.Context) (creds.Credential, bool, error) { + return s.cred, true, nil +} + +// An identity credential (KindIdentity) has no home on a direct SQL connection and +// must be refused, never landed in the password slot. +func TestResolveDSNCredentialRefusesIdentity(t *testing.T) { + src := staticSource{cred: creds.Credential{Value: "an-identity-token", Kind: creds.KindIdentity, Source: "test"}} + _, err := resolveDSNWithSources(context.Background(), baseDSN, src) + if err == nil || !strings.Contains(err.Error(), "identity") { + t.Fatalf("expected an identity-refusal error, got %v", err) + } +} + +// A credential carrying a username (a Vault dynamic user/password pair) places both. +func TestResolveDSNCredentialPlacesUsername(t *testing.T) { + src := staticSource{cred: creds.Credential{Value: "pw", Username: "dyn", Kind: creds.KindSecret}} + got, err := resolveDSNWithSources(context.Background(), baseDSN, src) + if err != nil { + t.Fatal(err) + } + if u := user(t, got); u != "dyn" { + t.Fatalf("user = %q, want dyn", u) + } + if pw := password(t, got); pw != "pw" { + t.Fatalf("password = %q, want pw", pw) + } +} + +// An empty username leaves the DSN's user untouched. +func TestResolveDSNCredentialEmptyUsernameKeepsDSNUser(t *testing.T) { + src := staticSource{cred: creds.Credential{Value: "pw", Username: "", Kind: creds.KindSecret}} + got, err := resolveDSNWithSources(context.Background(), baseDSN, src) + if err != nil { + t.Fatal(err) + } + if u := user(t, got); u != "bts" { + t.Fatalf("user = %q, want bts (DSN user preserved)", u) + } +} + +// End-to-end: a JSON envelope carrying a username places both user and password. +func TestResolveDSNCredentialEnvelopeUsernameEndToEnd(t *testing.T) { + t.Setenv("BEADS_PG_PASSWORD_COMMAND", `printf '{"token":"pw","username":"dyn"}'`) + t.Setenv("BEADS_PG_PASSWORD", "") + got, err := resolveDSNCredential(context.Background(), &configfile.Config{}, baseDSN) + if err != nil { + t.Fatal(err) + } + if u, pw := user(t, got), password(t, got); u != "dyn" || pw != "pw" { + t.Fatalf("(user,password) = (%q,%q), want (dyn,pw)", u, pw) + } +} diff --git a/internal/storage/postgres/dialect.go b/internal/storage/postgres/dialect.go new file mode 100644 index 000000000..1a5707f13 --- /dev/null +++ b/internal/storage/postgres/dialect.go @@ -0,0 +1,22 @@ +package postgres + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage/pgdialect" +) + +// pgDialect is the Postgres SQL flavor: it opens a *sql.DB whose driver +// translates bd's canonical (MySQL-dialect) SQL to Postgres, and pins the +// workspace schema via search_path. +type pgDialect struct { + dsn string + searchPath string +} + +func (d pgDialect) Name() string { return "postgres" } + +func (d pgDialect) Open(_ context.Context) (*sql.DB, error) { + return pgdialect.Open(d.dsn, d.searchPath) +} diff --git a/internal/storage/postgres/fromconfig.go b/internal/storage/postgres/fromconfig.go new file mode 100644 index 000000000..d43f224cf --- /dev/null +++ b/internal/storage/postgres/fromconfig.go @@ -0,0 +1,68 @@ +package postgres + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/pgdialect" +) + +// NewFromConfig opens the Postgres backend for a workspace. It reads the base DSN +// and per-workspace schema from .beads/metadata.json, resolves the password through +// the credential ladder (command > env > credentials file, fail-closed) and places +// it into the DSN β€” never persisting it β€” then applies the schema over a raw +// non-translating connection and returns the store. This is the factory arm cmd/bd +// dispatches to when metadata.json has backend="postgres". +func NewFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { + cfg, err := configfile.Load(beadsDir) + if err != nil { + return nil, fmt.Errorf("postgres: load config: %w", err) + } + dsn := cfg.GetPostgresDSN() + if dsn == "" { + return nil, fmt.Errorf("postgres: no DSN (set postgres_dsn in metadata.json, or BEADS_POSTGRES_URL)") + } + dsn, err = resolveDSNCredential(ctx, cfg, dsn) + if err != nil { + return nil, fmt.Errorf("postgres: resolve credential: %w", err) + } + schema := cfg.GetPostgresSchema() + if schema == "" { + return nil, fmt.Errorf("postgres: no schema (set postgres_schema in metadata.json)") + } + return Provision(ctx, dsn, schema) +} + +// Provision opens the Postgres backend from an explicit DSN and schema: it applies +// the schema (InitSchema β€” create-schema, DDL, config seeds, version stamp; all +// idempotent) over a raw non-translating connection, then returns the store over the +// translating driver. dsn may carry a password; it is used only to connect and is +// never persisted. `bd init` calls this directly (before metadata.json exists, using +// the flag-provided DSN); NewFromConfig calls it after resolving DSN/schema from +// metadata. DDL runs raw because the translating driver would mangle the $$-quoted +// function bodies and treat DDL as workload SQL. +func Provision(ctx context.Context, dsn, schema string) (storage.DoltStorage, error) { + if dsn == "" { + return nil, fmt.Errorf("postgres: empty DSN") + } + if schema == "" { + return nil, fmt.Errorf("postgres: empty schema") + } + raw, err := pgdialect.OpenRaw(dsn, schema) + if err != nil { + return nil, fmt.Errorf("postgres: open (raw): %w", err) + } + if err := InitSchema(ctx, raw, schema); err != nil { + _ = raw.Close() + return nil, fmt.Errorf("postgres: init schema: %w", err) + } + // Byte-order parity with Dolt depends on the database collation (text columns + // use the DB default; per-column COLLATE "C" breaks the shared recursive CTEs). + // Warn loudly when it is not code-point ordered, at init and on every open. + warnOnCollationDivergence(ctx, raw) + _ = raw.Close() + + return New(ctx, Config{DSN: dsn, Schema: schema}) +} diff --git a/internal/storage/postgres/repro_instr_test.go b/internal/storage/postgres/repro_instr_test.go new file mode 100644 index 000000000..6b1478c2e --- /dev/null +++ b/internal/storage/postgres/repro_instr_test.go @@ -0,0 +1,46 @@ +package postgres + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// Temporary red-team repro: create with NO explicit ID (default hash-ID path) +// which exercises GetAdaptiveIDLengthTx's INSTR() query. +func TestPGReproHashIDCreate(t *testing.T) { + url := os.Getenv("BEADS_PG_TEST_URL") + if url == "" { + t.Skip("BEADS_PG_TEST_URL not set") + } + schema := fmt.Sprintf("repro_%d", time.Now().UnixNano()) + ctx := context.Background() + + st, err := New(ctx, Config{DSN: url, Schema: schema}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer st.Close() + if err := InitSchema(ctx, st.DB(), schema); err != nil { + t.Fatalf("InitSchema: %v", err) + } + t.Cleanup(func() { + _, _ = st.DB().ExecContext(context.Background(), fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) + }) + if err := st.SetConfig(ctx, "issue_prefix", "repro"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + iss := &types.Issue{ + Title: "no explicit id", IssueType: types.IssueType("task"), + Status: types.Status("open"), Priority: 2, CreatedBy: "tester", + } + if err := st.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("CreateIssue(hash-id): %v", err) + } + t.Logf("created id=%s", iss.ID) +} diff --git a/internal/storage/postgres/schema.go b/internal/storage/postgres/schema.go new file mode 100644 index 000000000..8518b8fb0 --- /dev/null +++ b/internal/storage/postgres/schema.go @@ -0,0 +1,675 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "fmt" + "hash/fnv" + "regexp" + "strings" +) + +// schemaAdvisoryLockKey derives a stable Postgres advisory-lock key from a +// workspace schema name. Distinct schemas map to distinct keys so per-workspace +// InitSchema calls serialize against themselves but never against each other. +func schemaAdvisoryLockKey(schema string) int64 { + h := fnv.New64a() + _, _ = h.Write([]byte("bd.initschema:" + schema)) + return int64(h.Sum64()) +} + +// ddl is the Postgres backend schema, embedded verbatim from +// internal/storage/postgres/schema.sql. It is applied statement-by-statement by +// InitSchema because the pgx extended protocol forbids multiple statements per +// Exec. +const ddl = `-- bd Postgres backend schema. +-- Machine-translated from Dolt ` + "`" + `SHOW CREATE TABLE` + "`" + ` output at HEAD (all +-- migrations applied), using the conformance type map documented in +-- docs/architecture/storage-backends.md: +-- varchar/char/text/longtext -> text (see collation note below) +-- datetime -> timestamp(0) (naive whole-second UTC) +-- tinyint(1) -> smallint +-- json DEFAULT (json_object()) -> jsonb DEFAULT '{}' +-- int -> integer, bigint -> bigint +-- ON UPDATE CURRENT_TIMESTAMP -> (no trigger; the shared layer always assigns +-- updated_at explicitly on every UPDATE) +-- inline KEY / UNIQUE KEY -> CREATE [UNIQUE] INDEX after each table +-- All statements are idempotent (IF NOT EXISTS / OR REPLACE); run with +-- search_path pinned to the per-workspace schema. +-- Views ready_issues / blocked_issues intentionally omitted (unused). +-- +-- Collation note: Dolt's text columns are utf8mb4_0900_bin (byte / code-point +-- order). Per-column COLLATE "C" was dropped from this DDL because it breaks the +-- shared layer's parameter-seeded recursive CTEs (ERROR: recursive query column +-- has collation "default" in non-recursive term but collation "C" overall). Text +-- columns therefore use the database's default collation, so byte-order parity +-- with Dolt is a documented wedge gap that must be guaranteed at DEPLOYMENT time +-- by creating the database with LC_COLLATE 'C' (CREATE DATABASE ... TEMPLATE +-- template0 LC_COLLATE 'C' LC_CTYPE 'C'), NOT per-column. Under any glibc/ICU +-- en_US.* collation, every text ORDER BY the shared layer emits diverges from +-- Dolt's code-point order. + +-- ============================================================ issues + +CREATE TABLE IF NOT EXISTS issues ( + id text NOT NULL, + content_hash text, + title text NOT NULL, + description text NOT NULL, + design text NOT NULL, + acceptance_criteria text NOT NULL, + notes text NOT NULL, + status text NOT NULL DEFAULT 'open', + priority integer NOT NULL DEFAULT 2, + issue_type text NOT NULL DEFAULT 'task', + assignee text, + estimated_minutes integer, + created_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + created_by text DEFAULT '', + owner text DEFAULT '', + updated_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + closed_at timestamp(0), + closed_by_session text DEFAULT '', + external_ref text, + spec_id text, + compaction_level integer DEFAULT 0, + compacted_at timestamp(0), + compacted_at_commit text, + original_size integer, + sender text DEFAULT '', + ephemeral smallint DEFAULT 0, + wisp_type text DEFAULT '', + pinned smallint DEFAULT 0, + is_template smallint DEFAULT 0, + mol_type text DEFAULT '', + work_type text DEFAULT 'mutex', + source_system text DEFAULT '', + metadata jsonb DEFAULT '{}', + source_repo text DEFAULT '', + close_reason text DEFAULT '', + event_kind text DEFAULT '', + actor text DEFAULT '', + target text DEFAULT '', + payload text DEFAULT '', + await_type text DEFAULT '', + await_id text DEFAULT '', + timeout_ns bigint DEFAULT 0, + waiters text DEFAULT '', + hook_bead text DEFAULT '', + role_bead text DEFAULT '', + agent_state text DEFAULT '', + last_activity timestamp(0), + role_type text DEFAULT '', + rig text DEFAULT '', + due_at timestamp(0), + defer_until timestamp(0), + no_history smallint DEFAULT 0, + started_at timestamp(0), + is_blocked smallint NOT NULL DEFAULT 0, + lease_expires_at timestamp(0), + heartbeat_at timestamp(0), + row_lock bigint NOT NULL DEFAULT 0, + PRIMARY KEY (id) +); + +CREATE INDEX IF NOT EXISTS idx_issues_assignee ON issues (assignee); +CREATE INDEX IF NOT EXISTS idx_issues_created_at ON issues (created_at); +CREATE INDEX IF NOT EXISTS idx_issues_defer_until ON issues (defer_until); +CREATE INDEX IF NOT EXISTS idx_issues_external_ref ON issues (external_ref); +CREATE INDEX IF NOT EXISTS idx_issues_is_blocked ON issues (is_blocked, status); +CREATE INDEX IF NOT EXISTS idx_issues_issue_type ON issues (issue_type); +CREATE INDEX IF NOT EXISTS idx_issues_priority ON issues (priority); +CREATE INDEX IF NOT EXISTS idx_issues_spec_id ON issues (spec_id); +CREATE INDEX IF NOT EXISTS idx_issues_status_updated_at ON issues (status, updated_at); +CREATE INDEX IF NOT EXISTS idx_issues_lease ON issues (status, lease_expires_at); + +-- ============================================================ wisps + +CREATE TABLE IF NOT EXISTS wisps ( + id text NOT NULL, + content_hash text, + title text NOT NULL, + description text NOT NULL DEFAULT '', + design text NOT NULL DEFAULT '', + acceptance_criteria text NOT NULL DEFAULT '', + notes text NOT NULL DEFAULT '', + status text NOT NULL DEFAULT 'open', + priority integer NOT NULL DEFAULT 2, + issue_type text NOT NULL DEFAULT 'task', + assignee text, + estimated_minutes integer, + created_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + created_by text DEFAULT '', + owner text DEFAULT '', + updated_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + closed_at timestamp(0), + closed_by_session text DEFAULT '', + external_ref text, + spec_id text, + compaction_level integer DEFAULT 0, + compacted_at timestamp(0), + compacted_at_commit text, + original_size integer, + sender text DEFAULT '', + ephemeral smallint DEFAULT 0, + wisp_type text DEFAULT '', + pinned smallint DEFAULT 0, + is_template smallint DEFAULT 0, + mol_type text DEFAULT '', + work_type text DEFAULT 'mutex', + source_system text DEFAULT '', + metadata jsonb DEFAULT '{}', + source_repo text DEFAULT '', + close_reason text DEFAULT '', + event_kind text DEFAULT '', + actor text DEFAULT '', + target text DEFAULT '', + payload text DEFAULT '', + await_type text DEFAULT '', + await_id text DEFAULT '', + timeout_ns bigint DEFAULT 0, + waiters text DEFAULT '', + hook_bead text DEFAULT '', + role_bead text DEFAULT '', + agent_state text DEFAULT '', + last_activity timestamp(0), + role_type text DEFAULT '', + rig text DEFAULT '', + due_at timestamp(0), + defer_until timestamp(0), + no_history smallint DEFAULT 0, + started_at timestamp(0), + is_blocked smallint NOT NULL DEFAULT 0, + lease_expires_at timestamp(0), + heartbeat_at timestamp(0), + row_lock bigint NOT NULL DEFAULT 0, + PRIMARY KEY (id) +); + +CREATE INDEX IF NOT EXISTS idx_wisps_assignee ON wisps (assignee); +CREATE INDEX IF NOT EXISTS idx_wisps_created_at ON wisps (created_at); +CREATE INDEX IF NOT EXISTS idx_wisps_external_ref ON wisps (external_ref); +CREATE INDEX IF NOT EXISTS idx_wisps_is_blocked ON wisps (is_blocked, status); +CREATE INDEX IF NOT EXISTS idx_wisps_issue_type ON wisps (issue_type); +CREATE INDEX IF NOT EXISTS idx_wisps_priority ON wisps (priority); +CREATE INDEX IF NOT EXISTS idx_wisps_spec_id ON wisps (spec_id); +CREATE INDEX IF NOT EXISTS idx_wisps_status ON wisps (status); + +-- ============================================================ helper functions +-- No updated_at trigger: MySQL's ON UPDATE CURRENT_TIMESTAMP is intentionally +-- NOT reproduced. The shared layer assigns updated_at explicitly on every issues +-- / wisps UPDATE, including the is_blocked mark/unmark idiom that self-assigns +-- (SET updated_at = updated_at) precisely to SUPPRESS the auto-bump. A BEFORE +-- UPDATE trigger cannot see the statement's SET list, so it would stomp those +-- deliberately-suppressed writes to now() and corrupt synced timestamps on the +-- hottest dependency-state path; omitting it keeps PG identical to Dolt. + +-- bd_mysql_jsonkey normalizes a MySQL JSON path to the bare top-level key the +-- translator binds for dynamic metadata filters: '$.sprint' -> sprint, +-- '$."gc.routed_to"' -> gc.routed_to. The translator rewrites +-- JSON_EXTRACT(metadata, ?) -> metadata -> bd_mysql_jsonkey(?) and +-- JSON_UNQUOTE(JSON_EXTRACT(metadata, ?)) -> metadata ->> bd_mysql_jsonkey(?). +-- storage.JSONMetadataPath always quotes dotted keys into one top-level segment, +-- so a single-key ->/->> lookup is exactly conformant with Dolt. +CREATE OR REPLACE FUNCTION bd_mysql_jsonkey(p text) RETURNS text LANGUAGE sql IMMUTABLE AS $$ SELECT CASE WHEN p LIKE '$."%"' THEN substring(p from 4 for length(p)-4) ELSE substring(p from 3) END $$; + +-- ============================================================ labels + +CREATE TABLE IF NOT EXISTS labels ( + issue_id text NOT NULL, + label text NOT NULL, + PRIMARY KEY (issue_id, label), + CONSTRAINT fk_labels_issue FOREIGN KEY (issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_labels_label ON labels (label); + +-- ============================================================ wisp_labels + +CREATE TABLE IF NOT EXISTS wisp_labels ( + issue_id text NOT NULL, + label text NOT NULL, + PRIMARY KEY (issue_id, label), + CONSTRAINT fk_wisp_labels_issue FOREIGN KEY (issue_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_wisp_labels_label ON wisp_labels (label); + +-- ============================================================ dependencies + +CREATE TABLE IF NOT EXISTS dependencies ( + id text NOT NULL, + issue_id text NOT NULL, + type text NOT NULL DEFAULT 'blocks', + created_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + created_by text NOT NULL, + metadata jsonb DEFAULT '{}', + thread_id text DEFAULT '', + depends_on_issue_id text, + depends_on_wisp_id text, + depends_on_external text, + PRIMARY KEY (id), + CONSTRAINT fk_dep_issue FOREIGN KEY (issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT fk_dep_issue_target FOREIGN KEY (depends_on_issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT ck_dep_one_target CHECK ( + ((depends_on_issue_id IS NOT NULL)::integer + + (depends_on_wisp_id IS NOT NULL)::integer + + (depends_on_external IS NOT NULL)::integer) = 1 + ) +); + +CREATE INDEX IF NOT EXISTS idx_dep_external_target ON dependencies (depends_on_external); +CREATE INDEX IF NOT EXISTS idx_dep_issue_target ON dependencies (depends_on_issue_id); +CREATE INDEX IF NOT EXISTS idx_dep_type_external ON dependencies (type, depends_on_external); +CREATE INDEX IF NOT EXISTS idx_dep_type_issue ON dependencies (type, depends_on_issue_id); +CREATE INDEX IF NOT EXISTS idx_dep_type_wisp ON dependencies (type, depends_on_wisp_id); +CREATE INDEX IF NOT EXISTS idx_dep_wisp_target ON dependencies (depends_on_wisp_id); +CREATE INDEX IF NOT EXISTS idx_dependencies_issue ON dependencies (issue_id); +CREATE INDEX IF NOT EXISTS idx_dependencies_thread ON dependencies (thread_id); +-- MySQL unique keys treat NULLs as distinct; PG's default (NULLS DISTINCT) matches. +CREATE UNIQUE INDEX IF NOT EXISTS uk_dep_external_target ON dependencies (issue_id, depends_on_external); +CREATE UNIQUE INDEX IF NOT EXISTS uk_dep_issue_target ON dependencies (issue_id, depends_on_issue_id); +CREATE UNIQUE INDEX IF NOT EXISTS uk_dep_wisp_target ON dependencies (issue_id, depends_on_wisp_id); + +-- ============================================================ wisp_dependencies + +CREATE TABLE IF NOT EXISTS wisp_dependencies ( + id text NOT NULL, + issue_id text NOT NULL, + depends_on_issue_id text, + depends_on_wisp_id text, + depends_on_external text, + type text NOT NULL DEFAULT 'blocks', + created_at timestamp(0) DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + created_by text DEFAULT '', + metadata jsonb DEFAULT '{}', + thread_id text DEFAULT '', + PRIMARY KEY (id), + CONSTRAINT fk_wisp_dep_issue FOREIGN KEY (issue_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT fk_wisp_dep_issue_target FOREIGN KEY (depends_on_issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT fk_wisp_dep_wisp_target FOREIGN KEY (depends_on_wisp_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT ck_wisp_dep_one_target CHECK ( + ((depends_on_issue_id IS NOT NULL)::integer + + (depends_on_wisp_id IS NOT NULL)::integer + + (depends_on_external IS NOT NULL)::integer) = 1 + ) +); + +CREATE INDEX IF NOT EXISTS fk_wisp_dep_issue_target ON wisp_dependencies (depends_on_issue_id); +CREATE INDEX IF NOT EXISTS fk_wisp_dep_wisp_target ON wisp_dependencies (depends_on_wisp_id); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_type ON wisp_dependencies (type); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_external ON wisp_dependencies (type, depends_on_external); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_issue ON wisp_dependencies (type, depends_on_issue_id); +CREATE INDEX IF NOT EXISTS idx_wisp_dep_type_wisp ON wisp_dependencies (type, depends_on_wisp_id); +CREATE UNIQUE INDEX IF NOT EXISTS uk_wisp_dep_external_target ON wisp_dependencies (issue_id, depends_on_external); +CREATE UNIQUE INDEX IF NOT EXISTS uk_wisp_dep_issue_target ON wisp_dependencies (issue_id, depends_on_issue_id); +CREATE UNIQUE INDEX IF NOT EXISTS uk_wisp_dep_wisp_target ON wisp_dependencies (issue_id, depends_on_wisp_id); + +-- ============================================================ comments + +CREATE TABLE IF NOT EXISTS comments ( + id text NOT NULL, + issue_id text NOT NULL, + author text NOT NULL, + text text NOT NULL, + created_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + PRIMARY KEY (id), + CONSTRAINT fk_comments_issue FOREIGN KEY (issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_comments_created_at ON comments (created_at); +CREATE INDEX IF NOT EXISTS idx_comments_issue ON comments (issue_id); + +-- ============================================================ wisp_comments + +CREATE TABLE IF NOT EXISTS wisp_comments ( + id text NOT NULL, + issue_id text NOT NULL, + author text DEFAULT '', + text text NOT NULL, + created_at timestamp(0) DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + PRIMARY KEY (id), + CONSTRAINT fk_wisp_comments_issue FOREIGN KEY (issue_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_wisp_comments_issue ON wisp_comments (issue_id); + +-- ============================================================ events + +CREATE TABLE IF NOT EXISTS events ( + id text NOT NULL, + issue_id text NOT NULL, + event_type text NOT NULL, + actor text NOT NULL, + old_value text, + new_value text, + comment text, + created_at timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + PRIMARY KEY (id), + CONSTRAINT fk_events_issue FOREIGN KEY (issue_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_events_created_at ON events (created_at); +CREATE INDEX IF NOT EXISTS idx_events_issue ON events (issue_id); + +-- ============================================================ wisp_events + +CREATE TABLE IF NOT EXISTS wisp_events ( + id text NOT NULL, + issue_id text NOT NULL, + event_type text NOT NULL, + actor text DEFAULT '', + old_value text DEFAULT '', + new_value text DEFAULT '', + comment text DEFAULT '', + created_at timestamp(0) DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + PRIMARY KEY (id), + CONSTRAINT fk_wisp_events_issue FOREIGN KEY (issue_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_wisp_events_created_at ON wisp_events (created_at); +CREATE INDEX IF NOT EXISTS idx_wisp_events_issue ON wisp_events (issue_id); + +-- ============================================================ config + +CREATE TABLE IF NOT EXISTS config ( + key text NOT NULL, + value text NOT NULL, + PRIMARY KEY (key) +); + +-- ============================================================ metadata + +CREATE TABLE IF NOT EXISTS metadata ( + key text NOT NULL, + value text NOT NULL, + PRIMARY KEY (key) +); + +-- ============================================================ local_metadata + +CREATE TABLE IF NOT EXISTS local_metadata ( + key text NOT NULL, + value text NOT NULL DEFAULT '', + PRIMARY KEY (key) +); + +-- ============================================================ issue_counter + +CREATE TABLE IF NOT EXISTS issue_counter ( + prefix text NOT NULL, + last_id integer NOT NULL DEFAULT 0, + PRIMARY KEY (prefix) +); + +-- ============================================================ child_counters + +CREATE TABLE IF NOT EXISTS child_counters ( + parent_id text NOT NULL, + last_child integer NOT NULL DEFAULT 0, + PRIMARY KEY (parent_id), + CONSTRAINT fk_counter_parent FOREIGN KEY (parent_id) REFERENCES issues (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +-- ============================================================ wisp_child_counters + +CREATE TABLE IF NOT EXISTS wisp_child_counters ( + parent_id text NOT NULL, + last_child integer NOT NULL DEFAULT 0, + PRIMARY KEY (parent_id), + CONSTRAINT fk_wisp_child_counters_parent FOREIGN KEY (parent_id) REFERENCES wisps (id) ON DELETE CASCADE ON UPDATE CASCADE +); + +-- ============================================================ custom_statuses + +CREATE TABLE IF NOT EXISTS custom_statuses ( + name text NOT NULL, + category text NOT NULL DEFAULT 'unspecified', + PRIMARY KEY (name) +); + +-- ============================================================ custom_types + +CREATE TABLE IF NOT EXISTS custom_types ( + name text NOT NULL, + PRIMARY KEY (name) +); + +-- ============================================================ repo_mtimes + +CREATE TABLE IF NOT EXISTS repo_mtimes ( + repo_path text NOT NULL, + jsonl_path text NOT NULL, + mtime_ns bigint NOT NULL, + last_checked timestamp(0) NOT NULL DEFAULT (date_trunc('second', now() AT TIME ZONE 'utc')), + PRIMARY KEY (repo_path) +); + +CREATE INDEX IF NOT EXISTS idx_repo_mtimes_checked ON repo_mtimes (last_checked); +` + +// schemaVersion is stamped into the metadata table by InitSchema. A workspace +// carrying a different stamp is rejected: this proof-wedge ships no migrator. +const schemaVersion = "1" + +// schemaVersionKey lives in the metadata table (NOT config): config rows are +// surfaced by issueops.GetAllConfig / `bd config list`, so a version key there +// would leak a permanent diff against the Dolt oracle in differential runs. The +// metadata table is where the shared layer already keeps backend-internal keys. +const schemaVersionKey = "pg_schema_version" + +// schemaNameRe restricts schema names to a plain SQL identifier so they can be +// safely interpolated into CREATE SCHEMA (which cannot take a placeholder). +var schemaNameRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// InitSchema creates the per-workspace schema if absent, then applies the +// embedded DDL one statement at a time and stamps the schema version. +// +// db must be a RAW *sql.DB (pgdialect.OpenRaw), not the translating one: the DDL +// is already native Postgres and routing it through Translate mangles the +// $$-quoted function bodies and rewrites now(). Version-stamp queries below use +// native $n placeholders for the same reason. +// +// All work runs on a single connection whose search_path is pinned to the target +// schema, so DDL never silently lands in public when db is handed unpinned. The +// schema name is validated (not parameterizable in DDL) and every DDL statement +// is executed on its own, since pgx's extended protocol rejects multiple +// statements in a single Exec. +func InitSchema(ctx context.Context, db *sql.DB, schema string) error { + if !schemaNameRe.MatchString(schema) { + return fmt.Errorf("postgres: invalid schema name %q", schema) + } + conn, err := db.Conn(ctx) + if err != nil { + return fmt.Errorf("postgres: acquire connection: %w", err) + } + defer conn.Close() + + // InitSchema runs on EVERY open and re-applies idempotent DDL (CREATE OR + // REPLACE FUNCTION, CREATE INDEX IF NOT EXISTS, ...). Idempotent is not the + // same as concurrency-safe: concurrent opens rewriting the same pg catalog + // tuples collide with "tuple concurrently updated" (SQLSTATE XX000). When a + // gc controller drives this store, the dispatcher, session reconciler, and + // agent subprocesses open bd concurrently, so the race is routine and it + // surfaces as spurious bd failures (e.g. the reconciler's assigned-work probe + // failing and orphan-draining live sessions). Serialize DDL application per + // schema with a session advisory lock so only one initializer touches the + // catalog at a time; distinct schemas hash to distinct keys and never block + // each other. Held on this connection and released before it returns to the + // pool. + lockKey := schemaAdvisoryLockKey(schema) + if _, err := conn.ExecContext(ctx, `SELECT pg_advisory_lock($1)`, lockKey); err != nil { + return fmt.Errorf("postgres: acquire schema-init advisory lock: %w", err) + } + // The unlock must run even when ctx is already canceled: with a done ctx, + // database/sql returns before touching the driver, the connection goes back + // to the pool healthy, and its backend session keeps holding the advisory + // lock β€” wedging every later open of this schema. WithoutCancel guarantees + // the unlock is attempted; if it fails on the wire instead, the driver marks + // the connection bad, the backend session dies with it, and Postgres releases + // the lock anyway, so the error is safe to drop. + defer func() { + _, _ = conn.ExecContext(context.WithoutCancel(ctx), `SELECT pg_advisory_unlock($1)`, lockKey) + }() + + if _, err := conn.ExecContext(ctx, `CREATE SCHEMA IF NOT EXISTS "`+schema+`"`); err != nil { + return fmt.Errorf("postgres: create schema %q: %w", schema, err) + } + if _, err := conn.ExecContext(ctx, `SET search_path TO "`+schema+`"`); err != nil { + return fmt.Errorf("postgres: set search_path %q: %w", schema, err) + } + for _, stmt := range splitStatements(ddl) { + if _, err := conn.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("postgres: exec DDL statement: %w\n-- statement:\n%s", err, stmt) + } + } + fresh, err := stampSchemaVersion(ctx, conn) + if err != nil { + return err + } + // Seed the default config rows ONLY on first provision β€” exactly like Dolt's + // migration 0016, which runs once. InitSchema runs on EVERY open (NewFromConfig β†’ + // Provision β†’ InitSchema) for idempotency, so seeding unconditionally would + // resurrect any config key a user has `config unset`, diverging from Dolt. + if fresh { + if err := seedDefaultConfig(ctx, conn); err != nil { + return err + } + } + return nil +} + +// defaultConfigSeeds mirrors internal/storage/schema/migrations/0016_default_config.up.sql +// exactly β€” the config rows a fresh Dolt workspace materializes on init. Seeding the +// identical set (and no more) keeps `bd config list` byte-identical to the Dolt oracle; +// on Dolt these land via a migration, which the wedge does not run, so InitSchema seeds +// them directly. issue_prefix is intentionally absent: it is workspace-specific and the +// caller (bd init) seeds it via store.SetConfig, matching the Dolt path. +var defaultConfigSeeds = [][2]string{ + {"compaction_enabled", "false"}, + {"compact_tier1_days", "30"}, + {"compact_tier1_dep_levels", "2"}, + {"compact_tier2_days", "90"}, + {"compact_tier2_dep_levels", "5"}, + {"compact_tier2_commits", "100"}, + {"compact_batch_size", "50"}, + {"compact_parallel_workers", "5"}, + {"auto_compact_enabled", "false"}, +} + +// seedDefaultConfig inserts the migration-0016 default config rows using +// ON CONFLICT DO NOTHING (the Postgres equivalent of the migration's INSERT IGNORE). +// InitSchema calls it only on first provision; the ON CONFLICT guard additionally +// makes it safe against a concurrent first init. Runs on the search_path-pinned +// connection so it targets the workspace config table. +func seedDefaultConfig(ctx context.Context, conn *sql.Conn) error { + for _, kv := range defaultConfigSeeds { + if _, err := conn.ExecContext(ctx, + `INSERT INTO config (key, value) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, + kv[0], kv[1]); err != nil { + return fmt.Errorf("postgres: seed default config %q: %w", kv[0], err) + } + } + return nil +} + +// stampSchemaVersion records schemaVersion in the metadata table on first init +// and refuses to open a workspace written by a binary with a different schema +// version (the proof-wedge has no migrator). It returns fresh=true only when it +// stamped a brand-new schema (no version row yet) β€” the signal InitSchema uses to +// seed one-time data exactly once. Runs on the search_path-pinned connection so it +// resolves the workspace metadata table. +func stampSchemaVersion(ctx context.Context, conn *sql.Conn) (fresh bool, err error) { + var stored string + err = conn.QueryRowContext(ctx, `SELECT value FROM metadata WHERE key = $1`, schemaVersionKey).Scan(&stored) + switch { + case errors.Is(err, sql.ErrNoRows): + // Provision runs on every open, so concurrent first opens of the same + // workspace (e.g. the hosted server from two goroutines) can both land + // here. ON CONFLICT DO NOTHING lets both succeed; only the opener that + // actually inserted the row (RowsAffected==1) is fresh and seeds defaults. + res, ierr := conn.ExecContext(ctx, `INSERT INTO metadata (key, value) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, schemaVersionKey, schemaVersion) + if ierr != nil { + return false, fmt.Errorf("postgres: stamp schema version: %w", ierr) + } + if n, _ := res.RowsAffected(); n == 1 { + return true, nil + } + // Lost the race: re-read what the winner stored and version-check it. + if rerr := conn.QueryRowContext(ctx, `SELECT value FROM metadata WHERE key = $1`, schemaVersionKey).Scan(&stored); rerr != nil { + return false, fmt.Errorf("postgres: read schema version after conflict: %w", rerr) + } + if stored != schemaVersion { + return false, fmt.Errorf("postgres: workspace schema version %s, this binary requires %s β€” no migrator in the proof-wedge, recreate the workspace or use a matching binary", stored, schemaVersion) + } + return false, nil + case err != nil: + return false, fmt.Errorf("postgres: read schema version: %w", err) + case stored != schemaVersion: + return false, fmt.Errorf("postgres: workspace schema version %s, this binary requires %s β€” no migrator in the proof-wedge, recreate the workspace or use a matching binary", stored, schemaVersion) + default: + return false, nil + } +} + +// splitStatements splits the DDL into individual statements on top-level ';' +// boundaries. It ignores ';' that appears inside a $$-delimited body (the +// trigger function), inside a single-quoted string literal, or inside a '--' +// line comment; line comments are dropped so a ';' in prose never terminates a +// statement. Blank fragments are omitted. +func splitStatements(ddl string) []string { + var stmts []string + var buf strings.Builder + var inString, inDollar bool + n := len(ddl) + for i := 0; i < n; i++ { + c := ddl[i] + switch { + case inString: + buf.WriteByte(c) + if c == '\'' { + if i+1 < n && ddl[i+1] == '\'' { + buf.WriteByte(ddl[i+1]) + i++ + continue + } + inString = false + } + case inDollar: + if c == '$' && i+1 < n && ddl[i+1] == '$' { + buf.WriteString("$$") + inDollar = false + i++ + continue + } + buf.WriteByte(c) + default: + switch { + case c == '-' && i+1 < n && ddl[i+1] == '-': + for i < n && ddl[i] != '\n' { + i++ + } + // Leave i at the newline (or end); the loop's i++ advances past it. + case c == '$' && i+1 < n && ddl[i+1] == '$': + buf.WriteString("$$") + inDollar = true + i++ + case c == '\'': + inString = true + buf.WriteByte(c) + case c == ';': + if s := strings.TrimSpace(buf.String()); s != "" { + stmts = append(stmts, s) + } + buf.Reset() + default: + buf.WriteByte(c) + } + } + } + if s := strings.TrimSpace(buf.String()); s != "" { + stmts = append(stmts, s) + } + return stmts +} diff --git a/internal/storage/postgres/schema_test.go b/internal/storage/postgres/schema_test.go new file mode 100644 index 000000000..20f1abc34 --- /dev/null +++ b/internal/storage/postgres/schema_test.go @@ -0,0 +1,155 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "os" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/pgdialect" +) + +// TestInitSchema applies the embedded DDL into a throwaway schema on a live +// Postgres and asserts the core tables materialize. It is gated on +// BEADS_PG_TEST_URL (a pgx-parseable DSN) and skips otherwise. +func TestInitSchema(t *testing.T) { + url := os.Getenv("BEADS_PG_TEST_URL") + if url == "" { + t.Skip("BEADS_PG_TEST_URL not set; skipping Postgres schema test") + } + + schema := fmt.Sprintf("bd_schema_test_%d", time.Now().UnixNano()) + + // DDL runs over a RAW (non-translating) DB; assertions below use ? bindings + // and so need the translating DB. + raw, err := pgdialect.OpenRaw(url, schema) + if err != nil { + t.Fatalf("openraw: %v", err) + } + db, err := pgdialect.Open(url, schema) + if err != nil { + t.Fatalf("open: %v", err) + } + // Registered first so they run last (cleanups are LIFO): the schema drop + // below still has open connections when it runs. + t.Cleanup(func() { _ = raw.Close() }) + t.Cleanup(func() { _ = db.Close() }) + + ctx := context.Background() + + if err := InitSchema(ctx, raw, schema); err != nil { + t.Fatalf("InitSchema: %v", err) + } + t.Cleanup(func() { + if _, err := db.ExecContext(ctx, `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop schema %q: %v", schema, err) + } + }) + + for _, table := range []string{"issues", "dependencies"} { + if !tableExists(ctx, t, db, schema, table) { + t.Errorf("expected table %q to exist in schema %q", table, schema) + } + } +} + +func tableExists(ctx context.Context, t *testing.T, db *sql.DB, schema, table string) bool { + t.Helper() + var count int + err := db.QueryRowContext(ctx, + `SELECT count(*) FROM information_schema.tables WHERE table_schema = ? AND table_name = ?`, + schema, table).Scan(&count) + if err != nil { + t.Fatalf("query information_schema.tables for %q: %v", table, err) + } + return count > 0 +} + +// TestSchemaAdvisoryLockKey pins the advisory-lock key derivation. The key is a +// cross-process contract: every bd binary that opens the same workspace schema +// must derive the same key, or their InitSchema calls stop serializing against +// each other. The golden values fail loudly if the hash, prefix, or truncation +// ever changes β€” such a change would silently break lock interop with +// already-deployed binaries during a rolling upgrade. +func TestSchemaAdvisoryLockKey(t *testing.T) { + golden := map[string]int64{ + "beads": -7207352757236036547, + "public": -3420031902527778135, + } + for schema, want := range golden { + if got := schemaAdvisoryLockKey(schema); got != want { + t.Errorf("schemaAdvisoryLockKey(%q) = %d, want %d β€” changing the derivation breaks locking against deployed binaries", schema, got, want) + } + } + if a, b := schemaAdvisoryLockKey("workspace_a"), schemaAdvisoryLockKey("workspace_b"); a == b { + t.Errorf("distinct schemas derived the same advisory-lock key %d; per-workspace inits must not serialize against each other", a) + } +} + +// TestInitSchemaConcurrentOpens is the regression test for the catalog race the +// advisory lock fixes: InitSchema runs on EVERY open and re-applies idempotent +// DDL, and bd is a short-lived CLI, so a busy deployment runs many concurrent +// opens. Without per-schema serialization those opens rewrite the same pg +// catalog tuples and fail with "tuple concurrently updated" (SQLSTATE XX000). +// Each worker gets its own *sql.DB pool β€” its own backend session β€” to mimic +// separate bd processes, and all hammer InitSchema on the same fresh schema. +// Gated on BEADS_PG_TEST_URL like TestInitSchema. +func TestInitSchemaConcurrentOpens(t *testing.T) { + url := os.Getenv("BEADS_PG_TEST_URL") + if url == "" { + t.Skip("BEADS_PG_TEST_URL not set; skipping Postgres schema test") + } + + schema := fmt.Sprintf("bd_schema_race_%d", time.Now().UnixNano()) + ctx := context.Background() + + const workers = 8 + const opensPerWorker = 3 + + dbs := make([]*sql.DB, workers) + for i := range dbs { + raw, err := pgdialect.OpenRaw(url, schema) + if err != nil { + t.Fatalf("openraw %d: %v", i, err) + } + dbs[i] = raw + } + // Registered first so it runs last (cleanups are LIFO): the schema drop + // below still has open pools when it runs. + t.Cleanup(func() { + for _, db := range dbs { + _ = db.Close() + } + }) + t.Cleanup(func() { + if _, err := dbs[0].ExecContext(ctx, `DROP SCHEMA IF EXISTS "`+schema+`" CASCADE`); err != nil { + t.Errorf("drop schema %q: %v", schema, err) + } + }) + + start := make(chan struct{}) + errs := make(chan error, workers) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func(db *sql.DB) { + defer wg.Done() + <-start + for j := 0; j < opensPerWorker; j++ { + if err := InitSchema(ctx, db, schema); err != nil { + errs <- err + return + } + } + }(dbs[i]) + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("concurrent InitSchema: %v", err) + } +} diff --git a/internal/storage/postgres/seed_test.go b/internal/storage/postgres/seed_test.go new file mode 100644 index 000000000..a82fe1bc6 --- /dev/null +++ b/internal/storage/postgres/seed_test.go @@ -0,0 +1,57 @@ +package postgres + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/pgdialect" +) + +// TestSeedOnlyOnFirstProvision is the config_unset_seeded_roundtrip regression: the +// bts-rs differential oracle caught that seeding default config on EVERY open (InitSchema +// runs per NewFromConfig/Provision) resurrected any key a user had `config unset`. Seeding +// must happen only on the first provision, like Dolt's one-shot migration 0016. +func TestSeedOnlyOnFirstProvision(t *testing.T) { + url := os.Getenv("BEADS_PG_TEST_URL") + if url == "" { + t.Skip("BEADS_PG_TEST_URL not set; skipping live Postgres seed test") + } + schema := fmt.Sprintf("seedonce_%d", time.Now().UnixNano()) + ctx := context.Background() + t.Cleanup(func() { + if raw, err := pgdialect.OpenRaw(url, "public"); err == nil { + _, _ = raw.ExecContext(context.Background(), fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) + _ = raw.Close() + } + }) + + st, err := Provision(ctx, url, schema) + if err != nil { + t.Fatalf("Provision: %v", err) + } + // First provision seeds the default config rows. + if v, _ := st.GetConfig(ctx, "auto_compact_enabled"); v != "false" { + t.Fatalf("first provision: want seeded auto_compact_enabled=false, got %q", v) + } + // User unsets a seeded key. + if err := st.DeleteConfig(ctx, "auto_compact_enabled"); err != nil { + t.Fatalf("DeleteConfig: %v", err) + } + if v, _ := st.GetConfig(ctx, "auto_compact_enabled"); v != "" { + t.Fatalf("after unset: want empty, got %q", v) + } + _ = st.Close() + + // Re-open the SAME schema β€” InitSchema runs again. The unset key must STAY unset. + st2, err := Provision(ctx, url, schema) + if err != nil { + t.Fatalf("re-Provision: %v", err) + } + defer st2.Close() + if v, _ := st2.GetConfig(ctx, "auto_compact_enabled"); v != "" { + t.Fatalf("REGRESSION: re-provision resurrected an unset key: auto_compact_enabled=%q (want empty)", v) + } +} diff --git a/internal/storage/postgres/smoke_test.go b/internal/storage/postgres/smoke_test.go new file mode 100644 index 000000000..065c1eb5c --- /dev/null +++ b/internal/storage/postgres/smoke_test.go @@ -0,0 +1,104 @@ +package postgres + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/pgdialect" + "github.com/steveyegge/beads/internal/types" +) + +// TestPGSmoke drives the full stack β€” dialect translation + issueops delegation +// + the synchronous readiness strategy β€” against a live Postgres. It proves the +// wedge actually runs, not just compiles. Gated on BEADS_PG_TEST_URL. +func TestPGSmoke(t *testing.T) { + url := os.Getenv("BEADS_PG_TEST_URL") + if url == "" { + t.Skip("BEADS_PG_TEST_URL not set; skipping live Postgres smoke") + } + schema := fmt.Sprintf("smoke_%d", time.Now().UnixNano()) + ctx := context.Background() + + st, err := New(ctx, Config{DSN: url, Schema: schema}) + if err != nil { + t.Fatalf("New: %v", err) + } + defer st.Close() + // Schema DDL must run over a raw (non-translating) DB; st.DB() is the + // translating one used for the actual issueops workload below. + raw, err := pgdialect.OpenRaw(url, schema) + if err != nil { + t.Fatalf("OpenRaw: %v", err) + } + if err := InitSchema(ctx, raw, schema); err != nil { + _ = raw.Close() + t.Fatalf("InitSchema: %v", err) + } + _ = raw.Close() + t.Cleanup(func() { + _, _ = st.DB().ExecContext(context.Background(), fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, schema)) + }) + + if err := st.SetConfig(ctx, "issue_prefix", "smoke"); err != nil { + t.Fatalf("SetConfig: %v", err) + } + + mk := func(id, title string) { + iss := &types.Issue{ + ID: id, Title: title, IssueType: types.IssueType("task"), + Status: types.Status("open"), Priority: 2, CreatedBy: "tester", + } + if err := st.CreateIssue(ctx, iss, "tester"); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + } + mk("smoke-1", "first") + mk("smoke-2", "second") + + got, err := st.GetIssue(ctx, "smoke-1") + if err != nil || got == nil { + t.Fatalf("GetIssue(smoke-1): %v (got=%v)", err, got) + } + if got.Title != "first" { + t.Fatalf("GetIssue title = %q, want %q", got.Title, "first") + } + + readyIDs := func() []string { + rows, err := st.GetReadyWork(ctx, types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + ids := make([]string, len(rows)) + for i, r := range rows { + ids[i] = r.ID + } + return ids + } + + // Both open, no deps -> both ready. + if ids := readyIDs(); len(ids) != 2 { + t.Fatalf("before dep: ready=%v, want 2", ids) + } + + // smoke-2 depends on (is blocked by) smoke-1. + dep := &types.Dependency{IssueID: "smoke-2", DependsOnID: "smoke-1", Type: types.DependencyType("blocks"), CreatedBy: "tester"} + if err := st.AddDependency(ctx, dep, "tester"); err != nil { + t.Fatalf("AddDependency: %v", err) + } + + // Now only smoke-1 is ready (readiness projection ran in-tx). + if ids := readyIDs(); len(ids) != 1 || ids[0] != "smoke-1" { + t.Fatalf("after dep: ready=%v, want [smoke-1]", ids) + } + + // Close smoke-1 -> smoke-2 unblocks. + if err := st.CloseIssue(ctx, "smoke-1", "done", "tester", ""); err != nil { + t.Fatalf("CloseIssue: %v", err) + } + if ids := readyIDs(); len(ids) != 1 || ids[0] != "smoke-2" { + t.Fatalf("after close: ready=%v, want [smoke-2]", ids) + } +} diff --git a/internal/storage/postgres/store.go b/internal/storage/postgres/store.go new file mode 100644 index 000000000..9304023b6 --- /dev/null +++ b/internal/storage/postgres/store.go @@ -0,0 +1,87 @@ +// Package postgres is the Postgres backend for bd: the shared sqlkit.Store +// bundled with the Postgres dialect and a readiness strategy, plus the non-Dolt +// markers. It is a strategy bundle over the shared SQL-family implementation β€” +// no storage semantics are re-derived here. +package postgres + +import ( + "context" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/sqlkit" +) + +// Backend is the metadata.json backend identifier for this store. +const Backend = "postgres" + +// Store is the Postgres backend. It embeds *sqlkit.Store, so it inherits every +// core method and overrides only what Postgres specializes (nothing yet β€” the +// readiness projection and FOR UPDATE SKIP LOCKED claim are later strategy +// swaps that override GetReadyWork/ClaimReadyIssue without touching sqlkit). +type Store struct { + *sqlkit.Store + // unsupportedDoltStorage (generated, unsupported_gen.go) provides the + // capability surface Postgres does not implement β€” VC/history/remote/sync/ + // compaction/federation and a few non-demo core methods β€” each returning a + // typed *storage.ErrUnsupported. It is generated as the exact complement of + // *sqlkit.Store's method set, so there is no ambiguous-selector overlap. + unsupportedDoltStorage +} + +// Compile-time guard on the generated skip list: the composite backend (real +// *sqlkit.Store methods plus the typed-unsupported shell) must fully satisfy +// storage.DoltStorage. This fails the build on skip-list drift β€” a skipped +// method sqlkit does not actually implement (missing method), or a method both +// implemented and stubbed (ambiguous selector) β€” so no method can silently +// become "unsupported". +var _ storage.DoltStorage = (*Store)(nil) + +// Config configures a Postgres backend. +type Config struct { + DSN string // password merged from env at open; never persisted to metadata.json + Schema string // per-workspace schema, pinned via search_path +} + +// New builds the Postgres backend from its strategy bundle: the pg dialect +// (translating driver) and a no-op readiness strategy (the shared issueops +// helpers already maintain is_blocked targeted and in-transaction, matching the +// Dolt reference; SyncReadiness would only add a redundant per-write recompute). +func New(ctx context.Context, cfg Config) (*Store, error) { + base, err := sqlkit.New(ctx, sqlkit.Config{ + Dialect: pgDialect{dsn: cfg.DSN, searchPath: cfg.Schema}, + Readiness: sqlkit.NoopReadiness{}, + }) + if err != nil { + return nil, err + } + return &Store{Store: base}, nil +} + +// CommitGraphUnsupported is the negative maintenance marker (see +// storage.NonCommitGraphBackend): it tells cmd/bd's PostRun to skip Dolt-only +// maintenance. Only non-Dolt backends implement it, so the default path is +// unaffected. +func (s *Store) CommitGraphUnsupported() bool { return true } + +// CommitPending is a no-op on a non-version-controlled backend; write paths +// invoke it opportunistically and must not error. +func (s *Store) CommitPending(_ context.Context, _ string) (bool, error) { + return false, nil +} + +// Commit and its variants are no-ops on Postgres, NOT unsupported: a Dolt +// "commit" flushes the working set to history, but every sqlkit write already +// committed its own SQL transaction, so the data is durable. Command paths call +// store.Commit() opportunistically after writes; returning nil lets those writes +// stand instead of erroring on a history operation that has no PG meaning. +func (s *Store) Commit(_ context.Context, _ string) error { return nil } +func (s *Store) CommitWithConfig(_ context.Context, _ string) error { return nil } +func (s *Store) CommitMergeResolution(_ context.Context, _ string) error { return nil } + +// Compile-time proof the Postgres backend satisfies the full storage seam +// (core methods via *sqlkit.Store, the rest via the generated shell) and the +// negative maintenance marker. +var ( + _ storage.DoltStorage = (*Store)(nil) + _ storage.NonCommitGraphBackend = (*Store)(nil) +) diff --git a/internal/storage/postgres/unsupported.go b/internal/storage/postgres/unsupported.go new file mode 100644 index 000000000..6d1e45ae8 --- /dev/null +++ b/internal/storage/postgres/unsupported.go @@ -0,0 +1,24 @@ +package postgres + +import ( + "github.com/steveyegge/beads/internal/storage" +) + +// Regeneration: the postgres shell is the exact complement of *sqlkit.Store's +// method set. Regenerate unsupported_gen.go with the directive below after +// finalizing the skip list (the sqlkit-implemented ops β€” the integrator fills +// in ). gen's strict unmatched-skip validation then doubles as a drift +// tripwire against DoltStorage interface changes. +// +//go:generate go run ../unsupportedgen -pkg postgres -src .. -out unsupported_gen.go -type DoltStorage -skip AddDependency,AddIssueComment,AddLabel,ClaimIssue,ClaimReadyIssue,Close,CloseIssue,Commit,CommitMergeResolution,CommitPending,CommitWithConfig,CountDependencies,CountDependents,CountEvents,CountIssueComments,CountIssues,CountIssuesByGroup,CreateIssue,CreateIssues,DeleteConfig,DeleteIssue,DeleteIssues,DetectCycles,GetAllConfig,GetBlockedIssues,GetBlockingInfoForIssues,GetCommentCounts,GetCommentsForIssues,GetConfig,GetCustomStatuses,GetCustomStatusesDetailed,GetCustomTypes,GetDependencies,GetDependenciesWithMetadata,GetDependencyCounts,GetDependencyRecords,GetDependencyRecordsForIssues,GetDependencyTree,GetDependents,GetDependentsWithMetadata,GetEpicsEligibleForClosure,GetEvents,GetInfraTypes,GetIssue,GetIssueByExternalRef,GetIssueComments,GetIssuesByIDs,GetIssuesByLabel,GetLabels,GetLabelsForIssues,GetLocalMetadata,GetMetadata,GetNewlyUnblockedByClose,GetNextChildID,GetReadyWork,GetReadyWorkWithCounts,GetStaleIssues,GetStatistics,IsBlocked,IsInfraTypeCtx,IterBlockedIssues,IterDependenciesWithMetadata,IterDependentsWithMetadata,IterEvents,IterIssueComments,IterIssues,IterReadyWork,IterWisps,ListWisps,RemoveDependency,RemoveLabel,ReopenIssue,RunInTransaction,SearchIssues,SearchIssuesWithCounts,SetConfig,SetLocalMetadata,SetMetadata,UpdateIssue,UpdateIssueType,AddComment,ClearRepoMtime,CountDependentsByStatus,DeleteIssuesBySourceRepo,FindWispDependentsRecursive,GetAllDependencyRecords,GetAllEventsSince,GetMoleculeLastActivity,GetMoleculeProgress,GetRepoMtime,ImportIssueComment,IterAllDependencyRecords,IterAllEventsSince,PromoteFromEphemeral,SetRepoMtime,UpdateIssueID,CreateIssuesWithFullOptions,SlotClear,SlotGet,SlotSet,SearchIssueIDs,HeartbeatIssue,ReclaimExpiredLeases,UnclaimIssue + +// errUnsupported is the constructor every generated stub in unsupported_gen.go +// calls. It returns the backend-agnostic *storage.ErrUnsupported sentinel, whose +// Error() renders `operation %q not supported by the %s backend`. That message is +// correct for every stubbed op β€” the shell also stubs non-VC core methods +// (GetStatistics, DeleteIssues, ...) that a Dolt backend is not the remedy for β€” +// so it carries no history/version-control hint. Backend is the package const +// "postgres"; callers can errors.As/Is on the returned sentinel. +func errUnsupported(op string) error { + return &storage.ErrUnsupported{Op: op, Backend: Backend} +} diff --git a/internal/storage/postgres/unsupported_gen.go b/internal/storage/postgres/unsupported_gen.go new file mode 100644 index 000000000..ce228433e --- /dev/null +++ b/internal/storage/postgres/unsupported_gen.go @@ -0,0 +1,236 @@ +// Code generated by gen (postgres typed-unsupported shell); DO NOT EDIT. + +package postgres + +import ( + "context" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// unsupportedDoltStorage is the generated typed-unsupported shell for storage.DoltStorage. +// Every method returns *storage.ErrUnsupported; embed it and override the +// real slice. DO NOT hand-edit β€” regenerate with `go generate ./...`. +type unsupportedDoltStorage struct{} + +func (unsupportedDoltStorage) MergeSlotCreate(_ context.Context, _ string) (_ *types.Issue, err error) { + err = errUnsupported("MergeSlotCreate") + return +} + +func (unsupportedDoltStorage) MergeSlotCheck(_ context.Context) (_ *storage.MergeSlotStatus, err error) { + err = errUnsupported("MergeSlotCheck") + return +} + +func (unsupportedDoltStorage) MergeSlotAcquire(_ context.Context, _ string, _ string, _ bool) (_ *storage.MergeSlotResult, err error) { + err = errUnsupported("MergeSlotAcquire") + return +} + +func (unsupportedDoltStorage) MergeSlotRelease(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("MergeSlotRelease") + return +} + +func (unsupportedDoltStorage) Branch(_ context.Context, _ string) (err error) { + err = errUnsupported("Branch") + return +} + +func (unsupportedDoltStorage) Checkout(_ context.Context, _ string) (err error) { + err = errUnsupported("Checkout") + return +} + +func (unsupportedDoltStorage) CurrentBranch(_ context.Context) (_ string, err error) { + err = errUnsupported("CurrentBranch") + return +} + +func (unsupportedDoltStorage) DeleteBranch(_ context.Context, _ string) (err error) { + err = errUnsupported("DeleteBranch") + return +} + +func (unsupportedDoltStorage) ListBranches(_ context.Context) (_ []string, err error) { + err = errUnsupported("ListBranches") + return +} + +func (unsupportedDoltStorage) CommitExists(_ context.Context, _ string) (_ bool, err error) { + err = errUnsupported("CommitExists") + return +} + +func (unsupportedDoltStorage) GetCurrentCommit(_ context.Context) (_ string, err error) { + err = errUnsupported("GetCurrentCommit") + return +} + +func (unsupportedDoltStorage) Status(_ context.Context) (_ *storage.Status, err error) { + err = errUnsupported("Status") + return +} + +func (unsupportedDoltStorage) Log(_ context.Context, _ int) (_ []storage.CommitInfo, err error) { + err = errUnsupported("Log") + return +} + +func (unsupportedDoltStorage) Merge(_ context.Context, _ string) (_ []storage.Conflict, err error) { + err = errUnsupported("Merge") + return +} + +func (unsupportedDoltStorage) GetConflicts(_ context.Context) (_ []storage.Conflict, err error) { + err = errUnsupported("GetConflicts") + return +} + +func (unsupportedDoltStorage) ResolveConflicts(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("ResolveConflicts") + return +} + +func (unsupportedDoltStorage) History(_ context.Context, _ string) (_ []*storage.HistoryEntry, err error) { + err = errUnsupported("History") + return +} + +func (unsupportedDoltStorage) AsOf(_ context.Context, _ string, _ string) (_ *types.Issue, err error) { + err = errUnsupported("AsOf") + return +} + +func (unsupportedDoltStorage) Diff(_ context.Context, _ string, _ string) (_ []*storage.DiffEntry, err error) { + err = errUnsupported("Diff") + return +} + +func (unsupportedDoltStorage) AddRemote(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("AddRemote") + return +} + +func (unsupportedDoltStorage) RemoveRemote(_ context.Context, _ string) (err error) { + err = errUnsupported("RemoveRemote") + return +} + +func (unsupportedDoltStorage) HasRemote(_ context.Context, _ string) (_ bool, err error) { + err = errUnsupported("HasRemote") + return +} + +func (unsupportedDoltStorage) ListRemotes(_ context.Context) (_ []storage.RemoteInfo, err error) { + err = errUnsupported("ListRemotes") + return +} + +func (unsupportedDoltStorage) Push(_ context.Context) (err error) { + err = errUnsupported("Push") + return +} + +func (unsupportedDoltStorage) Pull(_ context.Context) (err error) { + err = errUnsupported("Pull") + return +} + +func (unsupportedDoltStorage) ForcePush(_ context.Context) (err error) { + err = errUnsupported("ForcePush") + return +} + +func (unsupportedDoltStorage) PushRemote(_ context.Context, _ string, _ bool) (err error) { + err = errUnsupported("PushRemote") + return +} + +func (unsupportedDoltStorage) PullRemote(_ context.Context, _ string) (err error) { + err = errUnsupported("PullRemote") + return +} + +func (unsupportedDoltStorage) Fetch(_ context.Context, _ string) (err error) { + err = errUnsupported("Fetch") + return +} + +func (unsupportedDoltStorage) PushTo(_ context.Context, _ string) (err error) { + err = errUnsupported("PushTo") + return +} + +func (unsupportedDoltStorage) PullFrom(_ context.Context, _ string) (_ []storage.Conflict, err error) { + err = errUnsupported("PullFrom") + return +} + +func (unsupportedDoltStorage) Sync(_ context.Context, _ string, _ string) (_ *storage.SyncResult, err error) { + err = errUnsupported("Sync") + return +} + +func (unsupportedDoltStorage) SyncStatus(_ context.Context, _ string) (_ *storage.SyncStatus, err error) { + err = errUnsupported("SyncStatus") + return +} + +func (unsupportedDoltStorage) AddFederationPeer(_ context.Context, _ *storage.FederationPeer) (err error) { + err = errUnsupported("AddFederationPeer") + return +} + +func (unsupportedDoltStorage) GetFederationPeer(_ context.Context, _ string) (_ *storage.FederationPeer, err error) { + err = errUnsupported("GetFederationPeer") + return +} + +func (unsupportedDoltStorage) ListFederationPeers(_ context.Context) (_ []*storage.FederationPeer, err error) { + err = errUnsupported("ListFederationPeers") + return +} + +func (unsupportedDoltStorage) RemoveFederationPeer(_ context.Context, _ string) (err error) { + err = errUnsupported("RemoveFederationPeer") + return +} + +func (unsupportedDoltStorage) CheckEligibility(_ context.Context, _ string, _ int) (_ bool, _ string, err error) { + err = errUnsupported("CheckEligibility") + return +} + +func (unsupportedDoltStorage) ApplyCompaction(_ context.Context, _ string, _ int, _ int, _ int, _ string) (err error) { + err = errUnsupported("ApplyCompaction") + return +} + +func (unsupportedDoltStorage) GetTier1Candidates(_ context.Context) (_ []*types.CompactionCandidate, err error) { + err = errUnsupported("GetTier1Candidates") + return +} + +func (unsupportedDoltStorage) GetTier2Candidates(_ context.Context) (_ []*types.CompactionCandidate, err error) { + err = errUnsupported("GetTier2Candidates") + return +} + +func (unsupportedDoltStorage) SnapshotIssue(_ context.Context, _ string, _ int) (err error) { + err = errUnsupported("SnapshotIssue") + return +} + +func (unsupportedDoltStorage) GetCompactionSnapshot(_ context.Context, _ string) (_ *types.IssueSnapshot, err error) { + err = errUnsupported("GetCompactionSnapshot") + return +} + +func (unsupportedDoltStorage) RestoreFromSnapshot(_ context.Context, _ string) (_ *types.IssueSnapshot, err error) { + err = errUnsupported("RestoreFromSnapshot") + return +} + +// NOTE: partial shell (104 methods skipped); the embedding composite must assert storage.DoltStorage itself. diff --git a/internal/storage/sqlbuild/sort.go b/internal/storage/sqlbuild/sort.go index 755b7b576..53bfd7825 100644 --- a/internal/storage/sqlbuild/sort.go +++ b/internal/storage/sqlbuild/sort.go @@ -70,9 +70,11 @@ func OrderByForColumns(sortBy string, sortDesc bool, col func(sortKey string) st if sortBy == "" || sortBy == "priority" { return fmt.Sprintf("ORDER BY %s %s, %s DESC, %s ASC", col("priority"), dir, col("created"), col("id")) } - // A nullable sort column (closed_at, assignee) treats NULL as lowest: first on ASC - // and last on DESC. Lead with an explicit (col IS NULL) key so the contract does not - // depend on a driver's default NULL ordering. NOT NULL columns keep the plain clause. + // A nullable sort column (closed_at, assignee) must order NULLs the MySQL way β€” NULL + // treated as lowest, so first on ASC and last on DESC β€” on every backend. MySQL/Dolt/ + // SQLite do this by default; Postgres defaults the opposite way (NULLS FIRST on DESC), + // so lead with an explicit (col IS NULL) key for those two. NOT NULL columns keep the + // plain clause (byte-identical to before). if sortBy == "closed" || sortBy == "assignee" { return fmt.Sprintf("ORDER BY (%s IS NULL) %s, %s %s, %s ASC", col(sortBy), flipDir(dir), col(sortBy), dir, col("id")) } @@ -100,7 +102,7 @@ func OrderBy(sortBy string, sortDesc bool, table string) string { // Less is the Go-side mirror of OrderBy for merge sorts over rows fetched // from separate queries (issues + wisps). It must order exactly the way the -// SQL does, including NULL-first ascending semantics for nullable columns; +// SQL does, including MySQL NULL-first semantics for nullable columns; // otherwise a post-merge limit cut keeps a different row set than SQL // selected. func Less(a, b *types.Issue, sortBy string, sortDesc bool) bool { diff --git a/internal/storage/sqlite/completeness_test.go b/internal/storage/sqlite/completeness_test.go index 04d727634..06af61c75 100644 --- a/internal/storage/sqlite/completeness_test.go +++ b/internal/storage/sqlite/completeness_test.go @@ -9,22 +9,24 @@ import ( "github.com/steveyegge/beads/internal/storage/conformance" ) -// legitimatelyUnsupported is the explicit set of storage.DoltStorage methods that -// SQLite deliberately does not implement, each with a reason. +// legitimatelyUnsupported is the EXPLICIT denominator of every storage.DoltStorage +// method the Postgres wedge deliberately does not implement, each with a reason. // // The interface-completeness gate (TestInterfaceCompleteness) asserts the // generated capability shell (unsupported_gen.go) contains EXACTLY these methods // β€” no more, no less. This is the guard the red-team had to stand in for: a -// method that silently resolves to the typed-unsupported shell fails this test +// method that silently resolves to the typed-unsupported shell (the class that +// bit us: DeleteIssues, GetCustomStatuses, the batch reads, …) fails this test // at build time instead of failing a user at runtime. Adding a method to the // shell without an entry here is a hard error that forces the triage question: // "implement it in sqlkit, or record here why the wedge legitimately omits it." var legitimatelyUnsupported = map[string]string{ - // VersionControl β€” SQLite has no Dolt commit graph. + // VersionControl β€” the Dolt commit graph. The wedge's whole premise is + // "bd on Postgres, history/version-control simply absent". "Branch": "VC: Dolt branch", "Checkout": "VC: Dolt checkout", "CurrentBranch": "VC", "DeleteBranch": "VC", "ListBranches": "VC", - // Commit/CommitWithConfig/CommitMergeResolution are no-ops on SQLite because - // data is already durable per transaction; sqlite.Store implements them. + // Commit/CommitWithConfig/CommitMergeResolution are NO-OPS on Postgres (data + // is already durable per-tx), not unsupported β€” implemented on postgres.Store. "CommitExists": "VC", "GetCurrentCommit": "VC: Dolt hash", "Status": "VC", "Log": "VC: dolt_log", "Merge": "VC", "GetConflicts": "VC", "ResolveConflicts": "VC", diff --git a/internal/storage/sqlite/dsn.go b/internal/storage/sqlite/dsn.go index 672756f7d..50cfd9a03 100644 --- a/internal/storage/sqlite/dsn.go +++ b/internal/storage/sqlite/dsn.go @@ -13,8 +13,9 @@ import "strings" // inter-process contention. // - _txlock=immediate: writers take the write lock up front and fail fast rather // than deadlocking mid-transaction on lock upgrade. -// - case_sensitive_like(1): LIKE parity with Dolt's binary table collation. -// SQLite's default LIKE is +// - case_sensitive_like(1): LIKE parity with the other backends. Dolt/MySQL use +// the utf8mb4_0900_bin table collation and Postgres uses collation "C", so +// `id LIKE 'bd-A%'` is case-sensitive there; SQLite's default LIKE is // ASCII-case-INsensitive, which silently diverged on raw-cased operands // (--id-prefix filtering, parent-descendant `id LIKE parent || '.%'`, // GetNextChildID's child scan). Intentionally case-insensitive searches are @@ -27,12 +28,14 @@ import "strings" // counts mega-query renders (translated to strftime by sqlitedialect) returns NULL and // deps_json exposes a zero created_at instead of the real timestamp. datetime stores // "2026-07-09 12:34:56", which strftime yields as "2026-07-09T12:34:56Z", giving the -// same whole-second granularity as Dolt datetime(0) and unifying bound timestamps +// same whole-second GRANULARITY as Dolt/MySQL datetime(0) and unifying bound timestamps // with the DEFAULT CURRENT_TIMESTAMP rows into one sortable, strftime-parseable format. // -// Note the granularity matches but the sub-second rule does not: Go's Format truncates -// the fraction, whereas Dolt datetime(0) rounds half-up. This accepted divergence is -// covered by conformance.testAuditImportCommentSubSecond. +// Note the granularity matches but the sub-second RULE does not: Go's Format truncates +// the fraction, whereas Dolt/MySQL datetime(0) round half-up (Postgres keeps microseconds). +// This sub-second divergence is an accepted, documented waiver β€” the portable contract is +// a whole-second round-trip (see conformance.testAuditImportCommentSubSecond); forcing +// round-down parity would only discard the SQL backends' native precision. func dsn(path string) string { if strings.HasPrefix(path, "file:") { return path diff --git a/internal/storage/sqlite/schema.go b/internal/storage/sqlite/schema.go index 52d077b5b..8087532e8 100644 --- a/internal/storage/sqlite/schema.go +++ b/internal/storage/sqlite/schema.go @@ -9,7 +9,7 @@ import ( "strings" ) -// ddl is the SQLite backend schema, embedded from schema.sql. It is the Dolt +// ddl is the SQLite backend schema, embedded from schema.sql. It is the Dolt/MySQL // schema with SQLite adjustments: ENGINE/CHARSET/COLLATE stripped, inline KEY/UNIQUE // KEY lifted to CREATE INDEX (SQLite forbids inline non-PK indexes), ON UPDATE // CURRENT_TIMESTAMP dropped (unsupported), json β†’ text, and prefix-index lengths diff --git a/internal/storage/sqlite/unsupported.go b/internal/storage/sqlite/unsupported.go index 6c9971684..5c92264b2 100644 --- a/internal/storage/sqlite/unsupported.go +++ b/internal/storage/sqlite/unsupported.go @@ -4,8 +4,8 @@ import "github.com/steveyegge/beads/internal/storage" //go:generate go run ../unsupportedgen -pkg sqlite -src .. -out unsupported_gen.go -type DoltStorage -skip AddDependency,AddIssueComment,AddLabel,ClaimIssue,ClaimReadyIssue,Close,CloseIssue,Commit,CommitMergeResolution,CommitPending,CommitWithConfig,CountDependencies,CountDependents,CountEvents,CountIssueComments,CountIssues,CountIssuesByGroup,CreateIssue,CreateIssues,DeleteConfig,DeleteIssue,DeleteIssues,DetectCycles,GetAllConfig,GetBlockedIssues,GetBlockingInfoForIssues,GetCommentCounts,GetCommentsForIssues,GetConfig,GetCustomStatuses,GetCustomStatusesDetailed,GetCustomTypes,GetDependencies,GetDependenciesWithMetadata,GetDependencyCounts,GetDependencyRecords,GetDependencyRecordsForIssues,GetDependencyTree,GetDependents,GetDependentsWithMetadata,GetEpicsEligibleForClosure,GetEvents,GetInfraTypes,GetIssue,GetIssueByExternalRef,GetIssueComments,GetIssuesByIDs,GetIssuesByLabel,GetLabels,GetLabelsForIssues,GetLocalMetadata,GetMetadata,GetNewlyUnblockedByClose,GetNextChildID,GetReadyWork,GetReadyWorkWithCounts,GetStaleIssues,GetStatistics,IsBlocked,IsInfraTypeCtx,IterBlockedIssues,IterDependenciesWithMetadata,IterDependentsWithMetadata,IterEvents,IterIssueComments,IterIssues,IterReadyWork,IterWisps,ListWisps,RemoveDependency,RemoveLabel,ReopenIssue,RunInTransaction,SearchIssues,SearchIssuesWithCounts,SetConfig,SetLocalMetadata,SetMetadata,UpdateIssue,UpdateIssueType,AddComment,ClearRepoMtime,CountDependentsByStatus,DeleteIssuesBySourceRepo,FindWispDependentsRecursive,GetAllDependencyRecords,GetAllEventsSince,GetMoleculeLastActivity,GetMoleculeProgress,GetRepoMtime,ImportIssueComment,IterAllDependencyRecords,IterAllEventsSince,PromoteFromEphemeral,SetRepoMtime,UpdateIssueID,CreateIssuesWithFullOptions,SlotClear,SlotGet,SlotSet,SearchIssueIDs,HeartbeatIssue,ReclaimExpiredLeases,UnclaimIssue -// errUnsupported returns the backend-agnostic *storage.ErrUnsupported sentinel for -// operations that are specific to Dolt's version-control capabilities. +// errUnsupported returns the backend-agnostic *storage.ErrUnsupported sentinel; Backend +// is "sqlite". Same allowlist as postgres/mysql (identical sqlkit base + overrides). func errUnsupported(op string) error { return &storage.ErrUnsupported{Op: op, Backend: Backend} } diff --git a/internal/storage/sqlitedialect/driver.go b/internal/storage/sqlitedialect/driver.go index 75e825732..078ceadad 100644 --- a/internal/storage/sqlitedialect/driver.go +++ b/internal/storage/sqlitedialect/driver.go @@ -9,8 +9,8 @@ import ( ) // Open returns a *sql.DB over modernc.org/sqlite whose Prepare translates bd's -// canonical MySQL SQL to SQLite (see Translate). It exposes no Execer/Queryer fast -// path, so database/sql routes every statement through Prepare. +// canonical MySQL SQL to SQLite (see Translate). Like pgdialect it exposes no +// Execer/Queryer fast path, so database/sql routes every statement through Prepare. // The caller supplies foreign-key enforcement etc. via DSN pragmas. func Open(dsn string) (*sql.DB, error) { return sql.OpenDB(&connector{dsn: dsn}), nil diff --git a/internal/storage/sqlitedialect/translate.go b/internal/storage/sqlitedialect/translate.go index 2c20516bd..ec186068d 100644 --- a/internal/storage/sqlitedialect/translate.go +++ b/internal/storage/sqlitedialect/translate.go @@ -1,7 +1,11 @@ -// Package sqlitedialect translates bd's canonical MySQL-dialect SQL to SQLite and -// wraps the modernc.org/sqlite driver. SQLite accepts many of the canonical forms -// natively; this package rewrites the remaining INSERT, upsert, function, and UPDATE -// forms. It stays isolated so shared issueops/sqlkit code remains backend-neutral. +// Package sqlitedialect translates bd's canonical (MySQL-dialect) SQL to SQLite and +// wraps the modernc.org/sqlite driver. SQLite is closer to MySQL than Postgres is β€” +// backtick identifiers, `?` placeholders, boolβ†’integer binding, INSTR, json_object, +// REPLACE INTO, and CURRENT_TIMESTAMP (UTC) all work natively β€” so the translation is +// a small subset of pgdialect's: INSERT IGNORE, the ON DUPLICATE KEY upsert (standard +// ON CONFLICT, shared with Postgres), CONCAT, and unqualified UPDATE SET targets. The +// string-scanning machinery (codeMask/rewriteFuncCalls/…) is shared in shape with +// pgdialect. Isolated here; shared issueops/sqlkit and other backends are untouched. package sqlitedialect import ( @@ -89,7 +93,7 @@ func rewriteInsertIgnore(sql string) string { return insertIgnoreRe.ReplaceAllString(sql, "INSERT OR IGNORE INTO") } -// --- ON DUPLICATE KEY UPDATE -> SQLite ON CONFLICT --- +// --- ON DUPLICATE KEY UPDATE -> ON CONFLICT (standard SQL, same syntax as Postgres) --- // onConflictTargets maps a table to its ON CONFLICT arbiter (the key MySQL's ON // DUPLICATE KEY infers). Only DO UPDATE upserts need an entry. @@ -344,7 +348,7 @@ func scanKeyword(s string, m []bool, from int, kw string) int { return -1 } -// --- string-scanning machinery used by the SQLite rewrites --- +// --- shared string-scanning machinery (shape mirrors pgdialect) --- func codeMask(sql string) []bool { b := []byte(sql) diff --git a/internal/storage/sqlkit/queries.go b/internal/storage/sqlkit/queries.go index b7f14d96b..ba4f8be3d 100644 --- a/internal/storage/sqlkit/queries.go +++ b/internal/storage/sqlkit/queries.go @@ -58,8 +58,8 @@ func (s *Store) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFil // CountReadyWork returns the total ready-work count for filter. It is identical // to len(GetReadyWorkWithCounts(filter with Limit=0)) but sizes the total with -// cheap indexed COUNT(*)s instead of re-running the counts mega-query, so SQLite -// satisfies storage.ReadyWorkCounter and +// cheap indexed COUNT(*)s instead of re-running the counts mega-query, so every +// SQL backend (Postgres/MySQL/SQLite) satisfies storage.ReadyWorkCounter and // `bd ready --json` prints "Showing X of N" without the unbounded pass. func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (int, error) { var n int diff --git a/internal/storage/sqlkit/store.go b/internal/storage/sqlkit/store.go index 1102ccf7d..638098c13 100644 --- a/internal/storage/sqlkit/store.go +++ b/internal/storage/sqlkit/store.go @@ -58,7 +58,7 @@ func New(ctx context.Context, cfg Config) (*Store, error) { // DB exposes the underlying pool for diagnostics/migrations. func (s *Store) DB() *sql.DB { return s.db } -// DialectName reports the backend's SQL dialect. +// DialectName reports the backend's dialect ("postgres", "sqlite", …). func (s *Store) DialectName() string { return s.dialect.Name() } // Close closes the pool. Idempotent. @@ -77,7 +77,9 @@ func (s *Store) Close() error { // would queue behind any in-flight writer and fail with SQLITE_BUSY after // busy_timeout, forfeiting WAL's reader/writer independence. TxOptions.ReadOnly // suppresses beginMode (plain deferred BEGIN), letting reads run concurrently -// with a writer. Every withReadTx callback is a pure issueops read. +// with a writer. On the server backends it maps to START TRANSACTION READ ONLY +// (mysql) / BEGIN READ ONLY (pgx), which every withReadTx callback satisfies: +// they are pure issueops reads. func (s *Store) withReadTx(ctx context.Context, fn func(tx *sql.Tx) error) error { if s.closed.Load() { return ErrStoreClosed diff --git a/internal/storage/sqlkit/strategy.go b/internal/storage/sqlkit/strategy.go index a10705bf2..3ac34cd16 100644 --- a/internal/storage/sqlkit/strategy.go +++ b/internal/storage/sqlkit/strategy.go @@ -1,12 +1,12 @@ -// Package sqlkit is the shared relational implementation used by the SQLite -// storage backend. It remains separated from the public storage interface as a -// strategy bundle: +// Package sqlkit is the single, shared SQL-family implementation of bd's storage +// semantics. Every SQL backend (Dolt, Postgres, SQLite, MySQL, …) is a strategy +// bundle over this one implementation: // // - Dialect β€” the SQL flavor (a translating driver or native emit) // - ReadinessStrategy β€” how the is_blocked projection is kept current // - ClaimStrategy β€” how a ready issue is atomically claimed // -// SQLite embeds *Store, inherits every method, and overrides only +// A concrete backend embeds *Store, inherits every method, and overrides only // the handful it specializes (a "method override") or adds capability-interface // methods for operations that don't exist in the core contract (e.g. Dolt // history). The query/graph/ordering/id-mint/cycle semantics live here once, in @@ -22,8 +22,9 @@ import ( ) // Dialect opens the backend's *sql.DB and names it. The shared layer emits -// canonical MySQL-dialect SQL inherited from the Dolt implementation; the SQLite -// dialect translates that text without changing the shared layer. +// canonical (MySQL-dialect) SQL; a Dialect whose driver translates that text to +// the backend's flavor is how a non-MySQL SQL engine joins the family without +// changing the shared layer (see internal/storage/pgdialect for Postgres). type Dialect interface { Name() string Open(ctx context.Context) (*sql.DB, error) diff --git a/internal/storage/unsupportedgen/gen_test.go b/internal/storage/unsupportedgen/gen_test.go index 460f03381..09b6cde74 100644 --- a/internal/storage/unsupportedgen/gen_test.go +++ b/internal/storage/unsupportedgen/gen_test.go @@ -25,7 +25,7 @@ import ( // shellBackends are the backends whose typed-unsupported shell this tool // generates. Each dir is relative to this package (internal/storage/unsupportedgen). -var shellBackends = []string{"sqlite"} +var shellBackends = []string{"postgres", "mysql", "sqlite"} func TestGeneratedShellsAreUpToDate(t *testing.T) { // storage package source, relative to this package dir (one level up). diff --git a/internal/storage/unsupportedgen/main.go b/internal/storage/unsupportedgen/main.go index 9a93e825b..4aed4efd9 100644 --- a/internal/storage/unsupportedgen/main.go +++ b/internal/storage/unsupportedgen/main.go @@ -12,7 +12,7 @@ // so the AST flatten resolves all embeds locally. // // Invoked via `//go:generate go run ../unsupportedgen -pkg -src .. ...` -// from a backend package dir (e.g. internal/storage/sqlite), whose CWD makes +// from a backend package dir (e.g. internal/storage/postgres), whose CWD makes // the storage package ".." β€” hence -src "..". package main @@ -65,7 +65,7 @@ func main() { // genConfig is the parameterized generation request. Its zero-flag defaults // (see defaultConfig) reproduce a backend's committed shell byte-for-byte; the // flags exist so the same generator can emit a partial shell for another -// package (e.g. sqlite) that embeds a real store alongside the generated one. +// package (e.g. postgres) that embeds a real store alongside the generated one. type genConfig struct { pkg string // -pkg : package clause of the generated file src string // -src : dir of the storage package to parse @@ -133,7 +133,7 @@ type pkgInfo struct { // runConfig is the parameterized generator. At defaultConfig() values it emits // a backend's committed shell byte-for-byte; -pkg/-type/-skip let it emit a -// partial shell for another package (e.g. sqlite) that embeds a real store +// partial shell for another package (e.g. postgres) that embeds a real store // alongside the generated one. func runConfig(cfg genConfig) error { info, err := parsePackage(cfg.src) diff --git a/internal/utils/id_parser.go b/internal/utils/id_parser.go index c3d8d8d15..75aa0c613 100644 --- a/internal/utils/id_parser.go +++ b/internal/utils/id_parser.go @@ -199,9 +199,11 @@ func ResolvePartialID(ctx context.Context, store storage.Storage, input string) return "", fmt.Errorf("no issue found matching %q", input) } - // Sort so the ambiguity error lists IDs deterministically. SearchIssues return - // order is not a contract for ambiguous matches, so sorting by ID pins the same - // message for every storage implementation. + // Sort so the ambiguity error lists IDs deterministically. matches is built + // in SearchIssues return order, whose primary sort is created_at DESC; that + // order is not stable across backends (Postgres timestamp(0) vs Dolt), so an + // unsorted list makes the same ambiguous input print in different orders on + // different storage. Sorting by ID pins the message regardless of backend. sort.Strings(matches) if len(matches) > 1 { diff --git a/open_backend_selection_test.go b/open_backend_selection_test.go deleted file mode 100644 index 659f1feea..000000000 --- a/open_backend_selection_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package beads_test - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/steveyegge/beads" -) - -func writeBackendMetadata(t *testing.T, backend string) string { - t.Helper() - beadsDir := filepath.Join(t.TempDir(), ".beads") - if err := os.MkdirAll(beadsDir, 0o750); err != nil { - t.Fatalf("create .beads: %v", err) - } - metadata := `{"backend":"` + backend + `"}` - if backend == "sqlite" { - // Current SQLite workspaces carry an explicit path marker. A bare - // backend:"sqlite" can be stale metadata from the earlier SQLite era - // and must not be used as a fresh-provisioning fixture (see PR #4740). - metadata = `{"backend":"sqlite","sqlite_path":"beads.db"}` - } - if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(metadata), 0o600); err != nil { - t.Fatalf("write metadata: %v", err) - } - return beadsDir -} - -func TestOpenBestAvailableRoutesSQLite(t *testing.T) { - beadsDir := writeBackendMetadata(t, "sqlite") - store, err := beads.OpenBestAvailable(context.Background(), beadsDir) - if err != nil { - t.Fatalf("OpenBestAvailable(SQLite): %v", err) - } - defer store.Close() - - if err := store.SetConfig(context.Background(), "routing-test", "sqlite"); err != nil { - t.Fatalf("write through SQLite store: %v", err) - } - if got, err := store.GetConfig(context.Background(), "routing-test"); err != nil || got != "sqlite" { - t.Fatalf("read through SQLite store = %q, %v; want sqlite, nil", got, err) - } - if _, err := os.Stat(filepath.Join(beadsDir, "beads.db")); err != nil { - t.Fatalf("SQLite database was not created: %v", err) - } - if _, err := os.Stat(filepath.Join(beadsDir, "embeddeddolt")); !os.IsNotExist(err) { - t.Fatalf("SQLite routing created embedded Dolt storage (stat error: %v)", err) - } -} - -func TestOpenBestAvailableRejectsRemovedBackends(t *testing.T) { - for _, backend := range []string{"postgres", "mysql"} { - t.Run(backend, func(t *testing.T) { - beadsDir := writeBackendMetadata(t, backend) - store, err := beads.OpenBestAvailable(context.Background(), beadsDir) - if store != nil { - _ = store.Close() - t.Fatalf("removed backend %q returned a store", backend) - } - if err == nil || !strings.Contains(err.Error(), "no longer supported") { - t.Fatalf("removed backend error = %v, want rollback explanation", err) - } - if !strings.Contains(err.Error(), "simple") || !strings.Contains(err.Error(), "resource") || !strings.Contains(err.Error(), "export") { - t.Fatalf("removed backend error lacks rationale or migration guidance: %v", err) - } - for _, name := range []string{"embeddeddolt", "dolt", "beads.db"} { - if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { - t.Fatalf("removed backend created %s (stat error: %v)", name, statErr) - } - } - }) - } -} - -func TestOpenBestAvailableRejectsUnknownBackend(t *testing.T) { - beadsDir := writeBackendMetadata(t, "mystery") - store, err := beads.OpenBestAvailable(context.Background(), beadsDir) - if store != nil { - _ = store.Close() - t.Fatal("unknown backend returned a store") - } - if err == nil || !strings.Contains(err.Error(), "not recognized") { - t.Fatalf("unknown backend error = %v, want fail-closed metadata guidance", err) - } - if !strings.Contains(err.Error(), "no storage database was opened or modified") { - t.Fatalf("unknown backend error lacks data-safety guarantee: %v", err) - } - for _, name := range []string{"embeddeddolt", "dolt", "beads.db"} { - if _, statErr := os.Stat(filepath.Join(beadsDir, name)); !os.IsNotExist(statErr) { - t.Fatalf("unknown backend created %s (stat error: %v)", name, statErr) - } - } -} - -func TestOpenBestAvailableRejectsCorruptMetadata(t *testing.T) { - beadsDir := filepath.Join(t.TempDir(), ".beads") - if err := os.MkdirAll(beadsDir, 0o750); err != nil { - t.Fatalf("create .beads: %v", err) - } - if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte("{"), 0o600); err != nil { - t.Fatalf("write metadata: %v", err) - } - - store, err := beads.OpenBestAvailable(context.Background(), beadsDir) - if store != nil { - _ = store.Close() - t.Fatal("corrupt metadata unexpectedly returned a store") - } - if err == nil || !strings.Contains(err.Error(), "metadata") { - t.Fatalf("corrupt metadata error = %v, want metadata load failure", err) - } - if _, statErr := os.Stat(filepath.Join(beadsDir, "embeddeddolt")); !os.IsNotExist(statErr) { - t.Fatalf("corrupt metadata created embedded Dolt storage (stat error: %v)", statErr) - } -} diff --git a/scripts/conformance.sh b/scripts/conformance.sh index a30623245..5937e6450 100755 --- a/scripts/conformance.sh +++ b/scripts/conformance.sh @@ -4,8 +4,17 @@ # # ./scripts/conformance.sh # -# Two tiers compare the maintained SQLite implementation against embedded Dolt: -# the in-process storage corpus, then the real-binary CLI corpus. +# Two tiers, both reading the backend registry (test/conformance/profiles.go for the +# E2E tier; the per-backend conformance_test.go files for the in-process tier). Add a +# backend = add a profile + a factory; both tiers pick it up. +# +# Backends that need an external service auto-skip when their env is unset: +# BEADS_PG_TEST_URL postgres://user:pass@host:port/db (enables the postgres backend) +# BEADS_PG_PASSWORD optional, if the password is not in the URL +# +# Optional deep gate (bts-rs 523-scenario differential oracle; needs the bts-rs +# checkout + ~50 min, so it is off by default and not part of the per-PR loop): +# CONFORMANCE_DEEP=1 BTS_RS_DIR=/path/to/bts-rs ./scripts/conformance.sh # set -euo pipefail cd "$(dirname "$0")/.." @@ -29,6 +38,37 @@ assert_conformance_passed() { rm -f "$log" } +# require_backend NAME -> success if NAME is listed (comma-separated) in +# BEADS_CONFORMANCE_REQUIRE. CI sets BEADS_CONFORMANCE_REQUIRE=postgres,mysql so a +# backend that self-skips because its BEADS_*_TEST_URL is missing becomes a hard +# failure instead of a silent green; unset locally, the backend self-skips as before. +require_backend() { + local want="$1" tok + local IFS=, + for tok in ${BEADS_CONFORMANCE_REQUIRE:-}; do + [ "$(printf '%s' "$tok" | tr -d '[:space:]')" = "$want" ] && return 0 + done + return 1 +} + +# run_sql_conformance NAME PKG RUN URLVAR +# Run one SQL backend's in-process conformance + wedge gates. When NAME is required +# (see require_backend), run verbose and assert the TestConformance parity oracle +# actually ran and passed β€” mirroring the embedded-Dolt reference guard so a missing +# BEADS_*_TEST_URL can't silently narrow coverage. Otherwise run plain and let the +# suite self-skip. URLVAR names the env var to set, surfaced in the failure hint. +run_sql_conformance() { + local name="$1" pkg="$2" run="$3" urlvar="$4" + if require_backend "$name"; then + local log + log="$(mktemp)" + CGO_ENABLED=1 go test -tags "$TAGS" -v "$pkg" -run "$run" | tee "$log" + assert_conformance_passed "$log" "$name backend (set $urlvar)" + else + CGO_ENABLED=1 go test -tags "$TAGS" "$pkg" -run "$run" + fi +} + echo "==> Tier 1: in-process store conformance + wedge gates" # The embedded-Dolt backend runs the full backend-agnostic suite (conformance.RunAll) as # the reference oracle every SQL backend is compared against. BEADS_TEST_EMBEDDED_DOLT=1 @@ -42,6 +82,26 @@ embedded_dolt_log="$(mktemp)" BEADS_TEST_EMBEDDED_DOLT=1 CGO_ENABLED=1 go test -tags "$TAGS" -v \ ./internal/storage/embeddeddolt/ -run TestConformance | tee "$embedded_dolt_log" assert_conformance_passed "$embedded_dolt_log" "embedded-Dolt reference (need BEADS_TEST_EMBEDDED_DOLT=1)" +# Each SQL backend now passes the FULL in-process store conformance suite (TestConformance +# = conformance.RunAll, the same ~40 behavior subtests the Dolt reference runs) β€” the only +# methods it does NOT implement are the genuinely-Dolt-only ones (version-control/remote/ +# sync/…), which RunAll does not exercise. Alongside RunAll: live smoke, the +# interface-completeness audit (shell == deferral allowlist β€” no SILENT unsupported) plus +# its behavioral complement (every allowlisted method returns typed ErrUnsupported), the +# seed-once regression, and the dialect corpus-PREPARE + password-redaction gates. All +# self-skip without BEADS_PG_TEST_URL β€” or hard-fail when BEADS_CONFORMANCE_REQUIRE +# lists postgres (see run_sql_conformance), so CI can't go green with the wedge absent. +run_sql_conformance postgres ./internal/storage/postgres/ \ + 'TestPGSmoke|TestInterfaceCompleteness|TestUnsupportedContract|TestConformance|TestSeedOnlyOnFirstProvision' \ + BEADS_PG_TEST_URL +CGO_ENABLED=1 go test -tags "$TAGS" ./internal/storage/pgdialect/ +# MySQL wedge gates (self-skip without BEADS_MYSQL_TEST_URL, or hard-fail when +# BEADS_CONFORMANCE_REQUIRE lists mysql); the dialect rewrite test (the is_blocked 1093 +# workaround) always runs. +run_sql_conformance mysql ./internal/storage/mysql/ \ + 'TestInterfaceCompleteness|TestUnsupportedContract|TestConformance|TestSeedOnlyOnFirstProvision' \ + BEADS_MYSQL_TEST_URL +CGO_ENABLED=1 go test -tags "$TAGS" ./internal/storage/mysqldialect/ # SQLite is embedded (pure-Go), always runs. CGO_ENABLED=1 go test -tags "$TAGS" ./internal/storage/sqlite/ \ -run 'TestInterfaceCompleteness|TestUnsupportedContract|TestConformance|TestSeedOnlyOnFirstProvision' @@ -50,4 +110,9 @@ CGO_ENABLED=1 go test -tags "$TAGS" ./internal/storage/sqlitedialect/ echo "==> Tier 2: end-to-end 'bd init' + CLI conformance (differential vs Dolt)" CGO_ENABLED=1 go test -tags "$TAGS e2e" ./test/conformance/ +if [[ "${CONFORMANCE_DEEP:-0}" == "1" ]]; then + echo "==> Deep: bts-rs 523-scenario differential oracle" + ./scripts/run-oracle-p.sh +fi + echo "==> conformance OK" diff --git a/scripts/run-oracle-p.sh b/scripts/run-oracle-p.sh new file mode 100755 index 000000000..f4ba3597f --- /dev/null +++ b/scripts/run-oracle-p.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Deep conformance gate: the bts-rs 523-scenario differential oracle, PG-vs-Dolt. +# Reference = bd in Dolt mode; candidate = the SAME bd via a thin flag-injector that +# runs native `bd init --backend=postgres`. On-demand only (needs the bts-rs checkout, +# a live Postgres, and ~50 min); invoked by scripts/conformance.sh when CONFORMANCE_DEEP=1. +# +# Env: +# BTS_RS_DIR path to the bts-rs checkout (required) +# BEADS_PG_TEST_URL postgres URL incl. password (required) +# BD path to a prebuilt bd binary (optional; built if unset) +# +set -euo pipefail +cd "$(dirname "$0")/.." +: "${BTS_RS_DIR:?set BTS_RS_DIR to the bts-rs checkout}" +: "${BEADS_PG_TEST_URL:?set BEADS_PG_TEST_URL (postgres URL incl. password)}" + +BD="${BD:-}" +if [[ -z "$BD" ]]; then + BD="$(mktemp -d)/bd" + echo "### building bd -> $BD" + CGO_ENABLED=1 go build -tags gms_pure_go -o "$BD" ./cmd/bd +fi + +# Thin flag-injector: native `bd init` does the metadata + schema + seeding; this only +# supplies the backend flags the backend-agnostic harness cannot know, plus a +# deterministic per-workspace schema and a default prefix so IDs match the reference. +WRAP="$(mktemp)" +cat > "$WRAP" </dev/null | psql "$BEADS_PG_TEST_URL" >/dev/null 2>&1 || true + +cd "$BTS_RS_DIR" +echo "### capturing goldens from bd-Dolt" +BTS_CATALOG=1 BTS_REFERENCE_BD="$BD" cargo run --release -q -p bts-conformance --bin capture_golden 2>&1 | tail -4 +echo "### scoring native bd-PG against fresh goldens" +BTS_CATALOG=1 BTS_CANDIDATE="$WRAP" BTS_DATABASE_URL="$BEADS_PG_TEST_URL" \ + cargo run --release -q -p bts-conformance --bin scoreboard 2>&1 | tail -30 +git -C "$BTS_RS_DIR" checkout crates/bts-conformance/testdata/golden >/dev/null 2>&1 || true +echo "### DONE" diff --git a/test/conformance/e2e_test.go b/test/conformance/e2e_test.go index d9506d0a2..5df2983df 100644 --- a/test/conformance/e2e_test.go +++ b/test/conformance/e2e_test.go @@ -20,9 +20,11 @@ import ( // scenario is a named sequence of bd CLI invocations. IDs are pinned with --id so // they match across backends; only timestamps/paths need normalization. // -// unordered marks a scenario whose JSON-array output has no contractual order. For -// these, array elements are sorted by id before comparison (matching the bts-rs -// oracle's multiset semantics for list output). +// unordered marks a scenario whose JSON-array output has no contractual order: bd +// sorts by priority then created_at, and Postgres's whole-second timestamps tie where +// Dolt's sub-second ones do not, so equal-priority items created in the same second +// can legitimately differ in order. For these, array elements are sorted by id before +// comparison (matching the bts-rs oracle's multiset semantics for list output). type scenario struct { name string steps [][]string @@ -31,7 +33,7 @@ type scenario struct { // corpus is the backend-agnostic CLI surface exercised end-to-end. It is deliberately // compact (the gc lifecycle + config), and grows over time; the bts-rs 523-scenario -// differential corpus exercises both maintained storage families. +// differential oracle (scripts/run-oracle-p.sh) remains the deep gate. var corpus = []scenario{ {name: "create-show", steps: [][]string{ {"create", "First task", "--id", "cf-a1", "-t", "task", "--json"}, @@ -73,7 +75,8 @@ var corpus = []scenario{ {"config", "unset", "custom.foo"}, {"config", "get", "custom.foo"}, }}, - // Deferred surfaces exercise the typed unsupported-operation contract. + // Deferred surfaces, present to exercise XFail classification against the + // postgres profile's allowlist (the reference passes these). {name: "stats", steps: [][]string{ {"create", "s", "--id", "cf-s1", "-t", "task"}, {"stats", "--json"}, @@ -140,10 +143,12 @@ var corpus = []scenario{ // comment UUIDv7 the CLI harness can't normalize, so it is not a CLI-differential.) // Case-variant config key round-trip. `config set` passes the key verbatim to // store.SetConfig (no ToLower) and namespaced keys bypass the recognized-key - // allowlist, so custom.TeamName reaches the store as-is. On both maintained - // backends the key column is case-sensitive, so + // allowlist, so custom.TeamName reaches the store as-is. On the embedded-Dolt + // reference the key column is case-SENSITIVE (verified at the store layer), so // `config get custom.teamname` does NOT see the value set under custom.TeamName, - // while `config get custom.TeamName` does. + // while `config get custom.TeamName` does. config set/get/unset all run on SQL + // backends. Divergence suspect: real MySQL's case-insensitive default collation + // would make custom.teamname resolve to the custom.TeamName row. {name: "config-case-variant-key", steps: [][]string{ {"config", "set", "custom.TeamName", "example-org"}, {"config", "get", "custom.teamname"}, @@ -159,8 +164,8 @@ var corpus = []scenario{ {"create", "childB", "-t", "task", "--parent", "cf-cp1", "--json"}, {"list", "--json"}, }}, - // (bd mol wisp list order is non-contractual for equal-second ties β€” Dolt and - // SQLite can differ on tie order in the nested wisps array, which the harness can't + // (bd mol wisp list order is non-contractual for equal-second ties β€” Dolt/PG vs + // MySQL/SQLite differ on tie order in the nested wisps array, which the harness can't // normalize; ListWisps is covered as a set at the store level.) // All cli-differential scenarios below were driven end-to-end against the // embedded-Dolt reference (bd built with -tags gms_pure_go, `bd init -p cf`) and @@ -305,14 +310,35 @@ func TestConformanceE2E(t *testing.T) { ref := Reference() cands := Candidates() if len(cands) == 0 { - t.Fatal("no candidate backends registered") + t.Skip("no candidate backends available (set BEADS_PG_TEST_URL for the postgres profile)") } - // Make coverage visible in test logs. + // Make coverage visible: sqlite's Available() is always true, so a run with + // BEADS_PG_TEST_URL/BEADS_MYSQL_TEST_URL unset compares only sqlite yet still + // passes green. Log the candidate set, and honor BEADS_CONFORMANCE_REQUIRE + // (comma-separated profile names) so CI hard-fails when an expected backend is + // absent rather than silently narrowing coverage. names := make([]string, len(cands)) for i, c := range cands { names[i] = c.Name } t.Logf("E2E differential candidates: %v (reference=%s)", names, ref.Name) + if req := strings.TrimSpace(os.Getenv("BEADS_CONFORMANCE_REQUIRE")); req != "" { + for _, want := range strings.Split(req, ",") { + if want = strings.TrimSpace(want); want == "" { + continue + } + found := false + for _, n := range names { + if n == want { + found = true + break + } + } + if !found { + t.Fatalf("BEADS_CONFORMANCE_REQUIRE lists %q but that candidate is unavailable (set its BEADS_*_TEST_URL)", want) + } + } + } for _, sc := range corpus { sc := sc t.Run(sc.name, func(t *testing.T) { @@ -339,6 +365,12 @@ func TestConformanceE2E(t *testing.T) { func runScenario(t *testing.T, bin string, p BackendProfile, sc scenario) []string { t.Helper() ws := &Workspace{Dir: t.TempDir()} + if p.NewHandle != nil { + ws.Handle = p.NewHandle() + } + if p.Teardown != nil { + t.Cleanup(func() { p.Teardown(ws) }) + } var env []string if p.Env != nil { env = p.Env(ws) @@ -408,10 +440,13 @@ var ( reRoleWarn = regexp.MustCompile(`(?m)^warning: beads\.role not configured \(GH#2950\)\.\n(?:^ (?:Fix|Or): .*\n?)*`) ) -// normalize removes cross-backend and cross-run noise: workspace path, timestamps, -// and random tip banners. Pinned IDs need no normalization. +// normalize removes cross-backend and cross-run noise: workspace path, schema handle, +// timestamps, and random tip banners. Pinned IDs need no normalization. func normalize(s string, ws *Workspace) string { s = strings.ReplaceAll(s, ws.Dir, "") + if ws.Handle != "" { + s = strings.ReplaceAll(s, ws.Handle, "") + } s = reZeroTS.ReplaceAllString(s, "") s = reTimestamp.ReplaceAllString(s, "") s = reTip.ReplaceAllString(s, "") diff --git a/test/conformance/handle.go b/test/conformance/handle.go new file mode 100644 index 000000000..df0b3b685 --- /dev/null +++ b/test/conformance/handle.go @@ -0,0 +1,61 @@ +package conformance + +import ( + "context" + "database/sql" + "fmt" + "os" + "sync/atomic" + + gomysql "github.com/go-sql-driver/mysql" + "github.com/steveyegge/beads/internal/storage/pgdialect" +) + +var handleSeq int64 + +// freshSchemaName mints a process-unique Postgres schema name. The pid keeps names +// distinct across concurrent runs and after a crashed run; teardown drops them. +func freshSchemaName() string { + return fmt.Sprintf("e2e_%d_%d", os.Getpid(), atomic.AddInt64(&handleSeq, 1)) +} + +// dropPostgresSchema tears down a workspace's schema. Best effort: a failure here +// only leaves a stray schema, which the next run's freshSchemaName avoids colliding +// with. Uses a raw (non-translating) connection since DROP SCHEMA is native DDL. +func dropPostgresSchema(ws *Workspace) { + if ws.Handle == "" { + return + } + url := os.Getenv("BEADS_PG_TEST_URL") + if url == "" { + return + } + raw, err := pgdialect.OpenRaw(url, "public") + if err != nil { + return + } + defer raw.Close() + _, _ = raw.ExecContext(context.Background(), fmt.Sprintf(`DROP SCHEMA IF EXISTS %q CASCADE`, ws.Handle)) +} + +// dropMySQLDatabase tears down a workspace's database. Best effort. +func dropMySQLDatabase(ws *Workspace) { + if ws.Handle == "" { + return + } + url := os.Getenv("BEADS_MYSQL_TEST_URL") + if url == "" { + return + } + cfg, err := gomysql.ParseDSN(url) + if err != nil { + return + } + cfg.DBName = "" + db, err := sql.Open("mysql", cfg.FormatDSN()) + if err != nil { + return + } + defer db.Close() + _, _ = db.ExecContext(context.Background(), "DROP DATABASE IF EXISTS `"+ws.Handle+"`") +} diff --git a/test/conformance/profiles.go b/test/conformance/profiles.go index 6a1735b31..626b3296e 100644 --- a/test/conformance/profiles.go +++ b/test/conformance/profiles.go @@ -1,19 +1,24 @@ // Package conformance is the end-to-end (real `bd` binary) conformance harness. It // complements the in-process store suite (internal/storage/conformance): that one // exercises a store object; this one exercises the whole CLI path β€” `bd init`, the -// factory, metadata.json, SQLite dialect translation over the real binary, and output +// factory, metadata.json, dialect translation over the real binary, and output // formatting β€” differentially against the Dolt reference. // // One declarative registry (Profiles) describes how to create and tear down an // isolated workspace for each backend. Adding a backend is one Profiles entry; the // runner and scripts/conformance.sh pick it up automatically. The suite is behind -// the `e2e` build tag because it shells out to a freshly built bd. +// the `e2e` build tag because it shells out to a freshly built bd and needs live +// backends for the non-default profiles. package conformance +import "os" + // Workspace is one isolated place a backend can be `bd init`-ed. Dir is the working -// directory; init creates its .beads directory. +// directory (its .beads/ is created by init); Handle is a backend-specific isolation +// id (a Postgres schema, a server database, …) used for teardown. type Workspace struct { - Dir string + Dir string + Handle string } // BackendProfile declares everything the E2E runner needs to init and isolate a @@ -32,6 +37,11 @@ type BackendProfile struct { InitArgs func(ws *Workspace) []string // Env returns extra environment (KEY=VALUE) for every bd invocation in ws. Env func(ws *Workspace) []string + // NewHandle mints a fresh isolation id (schema/db name) for a workspace. Empty + // for backends isolated purely by directory (embedded Dolt). + NewHandle func() string + // Teardown releases the isolation (drop schema/db). Best effort; may be nil. + Teardown func(ws *Workspace) // XFail lists scenarios this backend is known not to match the reference on, // each with a reason. They are reported as XFAIL (visible, never masked); an // XFAIL that starts matching is flagged so the list can only shrink. @@ -46,6 +56,43 @@ var Profiles = []BackendProfile{ Available: func() bool { return true }, InitArgs: func(*Workspace) []string { return nil }, }, + { + Name: "postgres", + Available: func() bool { return os.Getenv("BEADS_PG_TEST_URL") != "" }, + NewHandle: freshSchemaName, + InitArgs: func(ws *Workspace) []string { + return []string{ + "--backend=postgres", + "--pg-url=" + os.Getenv("BEADS_PG_TEST_URL"), + "--pg-schema=" + ws.Handle, + } + }, + Env: func(*Workspace) []string { + if pw := os.Getenv("BEADS_PG_PASSWORD"); pw != "" { + return []string{"BEADS_PG_PASSWORD=" + pw} + } + return nil + }, + Teardown: dropPostgresSchema, + }, + { + Name: "mysql", + Available: func() bool { return os.Getenv("BEADS_MYSQL_TEST_URL") != "" }, + NewHandle: freshSchemaName, // a valid MySQL database name + InitArgs: func(ws *Workspace) []string { + return []string{ + "--backend=mysql", + "--mysql-url=" + os.Getenv("BEADS_MYSQL_TEST_URL"), + "--mysql-database=" + ws.Handle, + } + }, + Env: func(*Workspace) []string { + // The server DSN (with password) is used for every command; the + // per-workspace database comes from metadata.json / --mysql-database. + return []string{"BEADS_MYSQL_URL=" + os.Getenv("BEADS_MYSQL_TEST_URL")} + }, + Teardown: dropMySQLDatabase, + }, { Name: "sqlite", Available: func() bool { return true }, // embedded (pure-Go modernc); always runs diff --git a/test/conformance/roundtrip_test.go b/test/conformance/roundtrip_test.go index 9cd862ba2..b2f1f9e3f 100644 --- a/test/conformance/roundtrip_test.go +++ b/test/conformance/roundtrip_test.go @@ -12,10 +12,12 @@ import ( "testing" ) -// The portable JSONL backup path β€” `bd export` -> git -> `bd import` β€” is SQLite's -// backup mechanism because Dolt-native CALL DOLT_BACKUP and commit hashes do not -// exist there. This test pins the manual round-trip so a regression in export or -// import cannot slip through CI. +// The portable JSONL backup path β€” `bd export` -> git -> `bd import` β€” is the only +// backup mechanism the non-Dolt backends have (Dolt-native CALL DOLT_BACKUP is +// unsupported on them, and auto-export is gated to Dolt because its change token is +// the Dolt commit hash). The manual round-trip is already lossless on every backend +// (verified live against real Postgres/MySQL); this test pins that so a regression in +// export or import cannot slip through CI. // // It is a self-consistency (idempotence) check, not a differential one, so it lives // outside the `corpus`: for each backend it seeds a rich fixture, exports, restores @@ -123,11 +125,18 @@ func restoreAndReexport(t *testing.T, bin string, p BackendProfile, exportJSONL return mustRead(t, out) } -// initRoundTripWorkspace creates an isolated temporary workspace and runs `bd init`. -// Prefix "rt" matches the pinned rt-* ids in the seed. +// initRoundTripWorkspace mints an isolated workspace for a backend (fresh temp dir plus +// a fresh handle/schema for the server backends) and runs `bd init`. Prefix "rt" matches +// the pinned rt-* ids in the seed. func initRoundTripWorkspace(t *testing.T, bin string, p BackendProfile) (*Workspace, []string) { t.Helper() ws := &Workspace{Dir: t.TempDir()} + if p.NewHandle != nil { + ws.Handle = p.NewHandle() + } + if p.Teardown != nil { + t.Cleanup(func() { p.Teardown(ws) }) + } var env []string if p.Env != nil { env = p.Env(ws)