diff --git a/cmd/bd/auto_import_upgrade.go b/cmd/bd/auto_import_upgrade.go index c26f55b70..9cf309fd9 100644 --- a/cmd/bd/auto_import_upgrade.go +++ b/cmd/bd/auto_import_upgrade.go @@ -19,11 +19,18 @@ type jsonlImporter interface { } // fallbackImporter is the function maybeAutoImportJSONL invokes for stores -// that do not implement jsonlImporter. It exists as a package-level variable -// so tests can substitute a counter and verify the top-level emptiness guard -// prevents the fallback path from running on a non-empty database. Production -// builds always use importFromLocalJSONLFull. -var fallbackImporter = importFromLocalJSONLFull +// that do not implement jsonlImporter (server-mode dolt). It exists as a +// package-level variable so tests can substitute a counter and verify the +// top-level emptiness guard prevents the fallback path from running on a +// non-empty database. +// +// Production builds use importFromLocalJSONLConflictSkip (GH#3955): this is +// upgrade-recovery into an empty DB, so insert-if-new and UPSERT are +// equivalent on the legitimate path — but if the emptiness guard above ever +// regresses again (cf. PR #3630), conflict-skip makes the fallback a +// harmless no-op instead of clobbering live rows. Explicit `bd import`, +// `bd bootstrap`, and `bd init --from-jsonl` are unaffected and keep UPSERT. +var fallbackImporter = importFromLocalJSONLConflictSkip type autoImportStamp struct { Size int64 `json:"size"` @@ -61,13 +68,16 @@ func writeAutoImportStamp(beadsDir string, info os.FileInfo) { // .beads/dolt/) to 1.0+ (which uses .beads/embeddeddolt/) don't appear to // lose their issues. See GH#2994. // -// The top-level emptiness guard (GetStatistics) protects both the embedded -// fast-path and the fallback path. The embedded jsonlImporter has its own -// in-transaction emptiness check as a concurrency-safe second line of defense; -// the fallback path's importFromLocalJSONLFull uses INSERT … ON DUPLICATE KEY -// UPDATE semantics under the hood, so without this guard a stale issues.jsonl -// would be re-imposed on top of live Dolt rows on every command, clobbering -// recent partial-update writes. +// The top-level emptiness guard (GetStatistics) is the primary +// protection for BOTH the embedded fast-path and the server-mode +// fallback. Defense in depth backs each path up: the embedded +// jsonlImporter has its own in-transaction emptiness check (and is +// also insert-if-new, GH#3955), and the fallback path imports via +// importFromLocalJSONLConflictSkip, which is insert-if-new rather than +// UPSERT. So if this guard ever regresses again (cf. PR #3630), a stale +// issues.jsonl can no longer be re-imposed on top of live Dolt rows — +// the worst case degrades to a harmless no-op instead of clobbering +// recent writes. // // The function is best-effort: failures are logged as warnings but do not // prevent the store from being used. diff --git a/cmd/bd/auto_import_upgrade_test.go b/cmd/bd/auto_import_upgrade_test.go index c93c977fe..43ba4206f 100644 --- a/cmd/bd/auto_import_upgrade_test.go +++ b/cmd/bd/auto_import_upgrade_test.go @@ -3,6 +3,7 @@ package main import ( + "context" "encoding/json" "fmt" "os" @@ -264,3 +265,123 @@ func bdListJSONAllowError(t *testing.T, bd, dir string, args ...string) []*types } return issues } + +// TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber asserts the +// GH#3955 Layer 2 guarantee against a real embedded store: the auto-import +// server-mode fallback importer (importFromLocalJSONLConflictSkip, the +// production fallbackImporter) never overwrites an existing issue row. +// +// We invoke it directly against a NON-empty store — deliberately modelling +// a regressed emptiness guard — and verify the live row is untouched while +// a brand-new id from the JSONL is still inserted. +func TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "cs") + + live := bdCreate(t, bd, dir, "live issue title", "--type", "task") + if live.ID == "" { + t.Fatal("expected a created issue ID") + } + + const newID = "cs-new-9999" + now := time.Now().UTC() + jsonlIssues := []types.Issue{ + // Mutated copy of the live issue: UPSERT would clobber it. + {ID: live.ID, Title: "CLOBBERED", Status: types.StatusClosed, Priority: 0, IssueType: types.TypeBug, CreatedAt: now, UpdatedAt: now}, + // Brand-new issue: must still be inserted. + {ID: newID, Title: "freshly imported", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, CreatedAt: now, UpdatedAt: now}, + } + var lines []string + for _, issue := range jsonlIssues { + b, err := json.Marshal(issue) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + lines = append(lines, string(b)) + } + jsonlPath := filepath.Join(beadsDir, "issues.jsonl") + if err := os.WriteFile(jsonlPath, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + + store := openStore(t, beadsDir, "cs") + ctx := context.Background() + + // Exactly the code path fallbackImporter points at; running it on a + // non-empty store models a guard regression (GH#3955 / PR #3630). + if _, err := importFromLocalJSONLConflictSkip(ctx, store, jsonlPath); err != nil { + t.Fatalf("importFromLocalJSONLConflictSkip: %v", err) + } + + got, err := store.GetIssue(ctx, live.ID) + if err != nil { + t.Fatalf("GetIssue(%s): %v", live.ID, err) + } + if got.Title != "live issue title" { + t.Errorf("live issue title was clobbered: got %q, want %q", got.Title, "live issue title") + } + if got.Status != types.StatusOpen { + t.Errorf("live issue status was clobbered: got %q, want %q", got.Status, types.StatusOpen) + } + + inserted, err := store.GetIssue(ctx, newID) + if err != nil { + t.Fatalf("brand-new issue %s was not inserted by conflict-skip import: %v", newID, err) + } + if inserted.Title != "freshly imported" { + t.Errorf("new issue title: got %q, want %q", inserted.Title, "freshly imported") + } +} + +// TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll is the +// negative control for GH#3955: conflict-skip must NOT regress the +// legitimate upgrade-recovery path. Against an empty store the fallback +// still imports every issue from the JSONL. +func TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + _, beadsDir, _ := bdInit(t, bd, "--prefix", "ce") + + now := time.Now().UTC() + jsonlIssues := []types.Issue{ + {ID: "ce-1", Title: "recovered one", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, CreatedAt: now, UpdatedAt: now}, + {ID: "ce-2", Title: "recovered two", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeBug, CreatedAt: now, UpdatedAt: now}, + } + var lines []string + for _, issue := range jsonlIssues { + b, err := json.Marshal(issue) + if err != nil { + t.Fatalf("marshal fixture: %v", err) + } + lines = append(lines, string(b)) + } + jsonlPath := filepath.Join(beadsDir, "issues.jsonl") + if err := os.WriteFile(jsonlPath, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + + store := openStore(t, beadsDir, "ce") + ctx := context.Background() + + res, err := importFromLocalJSONLConflictSkip(ctx, store, jsonlPath) + if err != nil { + t.Fatalf("importFromLocalJSONLConflictSkip: %v", err) + } + if res.Issues != 2 { + t.Errorf("expected 2 issues imported, got %d", res.Issues) + } + for _, id := range []string{"ce-1", "ce-2"} { + if _, err := store.GetIssue(ctx, id); err != nil { + t.Errorf("empty-DB recovery dropped issue %s: %v", id, err) + } + } +} diff --git a/cmd/bd/auto_import_upgrade_unit_test.go b/cmd/bd/auto_import_upgrade_unit_test.go index 1ace4ebf2..c9438da6a 100644 --- a/cmd/bd/auto_import_upgrade_unit_test.go +++ b/cmd/bd/auto_import_upgrade_unit_test.go @@ -240,3 +240,95 @@ func TestMaybeAutoImportJSONL_ChangedJSONLBypassesSuccessStamp(t *testing.T) { t.Fatalf("fallback importer invoked %d time(s), want 2 after JSONL changed", got) } } + +// captureOptsStore is a storage.DoltStorage that records the +// BatchCreateOptions handed to CreateIssuesWithFullOptions. Every other +// method panics (embedded nil interface); the import plumbing under test +// only touches CreateIssuesWithFullOptions, GetConfig and SetConfig. +type captureOptsStore struct { + storage.DoltStorage // nil — panics on any non-overridden method + prefix string + gotOpts storage.BatchCreateOptions +} + +func (c *captureOptsStore) CreateIssuesWithFullOptions(_ context.Context, _ []*types.Issue, _ string, opts storage.BatchCreateOptions) error { + c.gotOpts = opts + return nil +} + +func (c *captureOptsStore) GetConfig(_ context.Context, _ string) (string, error) { + return c.prefix, nil +} + +func (c *captureOptsStore) SetConfig(_ context.Context, _, _ string) error { + return nil +} + +// GetIssuesByIDs reports no existing rows so importIssuesCore's stale-import +// filter (filterStaleImportIssues) is a no-op and the import proceeds to +// CreateIssuesWithFullOptions, where the options under test are recorded. +func (c *captureOptsStore) GetIssuesByIDs(_ context.Context, _ []string) ([]*types.Issue, error) { + return nil, nil +} + +// TestImportIssuesCoreThreadsConflictSkip verifies the GH#3955 plumbing: +// ImportOptions.ConflictSkip maps onto storage.BatchCreateOptions.ConflictSkip, +// and the default (explicit `bd import`) path keeps UPSERT semantics. +func TestImportIssuesCoreThreadsConflictSkip(t *testing.T) { + issues := []*types.Issue{{ID: "unit-1", Title: "t", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}} + + t.Run("ConflictSkip true threads through", func(t *testing.T) { + s := &captureOptsStore{} + if _, err := importIssuesCore(context.Background(), "", s, issues, ImportOptions{SkipPrefixValidation: true, ConflictSkip: true}); err != nil { + t.Fatalf("importIssuesCore: %v", err) + } + if !s.gotOpts.ConflictSkip { + t.Fatalf("ConflictSkip not threaded into BatchCreateOptions: got %+v", s.gotOpts) + } + if !s.gotOpts.SkipPrefixValidation { + t.Errorf("SkipPrefixValidation should still thread through: got %+v", s.gotOpts) + } + }) + + t.Run("default keeps UPSERT", func(t *testing.T) { + s := &captureOptsStore{} + if _, err := importIssuesCore(context.Background(), "", s, issues, ImportOptions{SkipPrefixValidation: true}); err != nil { + t.Fatalf("importIssuesCore: %v", err) + } + if s.gotOpts.ConflictSkip { + t.Fatalf("explicit-import path must keep UPSERT (ConflictSkip=false); got true") + } + }) +} + +// TestAutoImportFallbackSeamUsesConflictSkip verifies which importer each +// caller is wired to: the auto-import server-mode fallback +// (importFromLocalJSONLConflictSkip, the fallbackImporter seam) requests +// conflict-skip, while importFromLocalJSONLFull — used by `bd bootstrap` +// and `bd init --from-jsonl` — keeps UPSERT. This is the scope boundary +// for GH#3955. +func TestAutoImportFallbackSeamUsesConflictSkip(t *testing.T) { + dir := t.TempDir() + writeAutoImportFixtureJSONL(t, dir) + jsonlPath := filepath.Join(dir, "issues.jsonl") + + t.Run("auto-import fallback uses conflict-skip", func(t *testing.T) { + s := &captureOptsStore{prefix: "unit"} + if _, err := importFromLocalJSONLConflictSkip(context.Background(), s, jsonlPath); err != nil { + t.Fatalf("importFromLocalJSONLConflictSkip: %v", err) + } + if !s.gotOpts.ConflictSkip { + t.Fatalf("auto-import fallback must set ConflictSkip=true; got %+v", s.gotOpts) + } + }) + + t.Run("explicit recovery path keeps UPSERT", func(t *testing.T) { + s := &captureOptsStore{prefix: "unit"} + if _, err := importFromLocalJSONLFull(context.Background(), s, jsonlPath); err != nil { + t.Fatalf("importFromLocalJSONLFull: %v", err) + } + if s.gotOpts.ConflictSkip { + t.Fatalf("bd bootstrap / init --from-jsonl must keep UPSERT (ConflictSkip=false); got true") + } + }) +} diff --git a/cmd/bd/import_shared.go b/cmd/bd/import_shared.go index 71b26e36a..077ecad2e 100644 --- a/cmd/bd/import_shared.go +++ b/cmd/bd/import_shared.go @@ -25,6 +25,11 @@ type ImportOptions struct { DeletionIDs []string SkipPrefixValidation bool ProtectLocalExportIDs map[string]time.Time + // ConflictSkip makes the import insert-if-new instead of UPSERT: an + // issue whose ID already exists is left untouched. Set only by the + // auto-import upgrade-recovery fallback (GH#3955); explicit `bd import` + // leaves this false and keeps UPSERT semantics. + ConflictSkip bool } // ImportResult describes what an import operation did. @@ -66,6 +71,7 @@ func importIssuesCore(ctx context.Context, _ string, store storage.DoltStorage, err = store.CreateIssuesWithFullOptions(ctx, issues, getActorWithGit(), storage.BatchCreateOptions{ OrphanHandling: storage.OrphanAllow, SkipPrefixValidation: opts.SkipPrefixValidation, + ConflictSkip: opts.ConflictSkip, SkipDependencyValidationErrors: true, OnSkippedDependency: func(issueID, dependsOnID, reason string) { skipped := fmt.Sprintf("%s -> %s: %s", issueID, dependsOnID, reason) @@ -251,10 +257,28 @@ func parseJSONLFile(path string) ([]*types.Issue, map[string]string, error) { return issues, configEntries, nil } -// importFromLocalJSONLFull imports issues and memories from a local JSONL file. -// It detects memory records (lines with "_type":"memory") and imports them -// via SetConfig, while routing regular issue records through the normal path. +// importFromLocalJSONLFull imports issues and memories from a local JSONL file +// using UPSERT semantics (an existing issue row is overwritten). Used by the +// explicit recovery paths: `bd bootstrap` and `bd init --from-jsonl`. func importFromLocalJSONLFull(ctx context.Context, store storage.DoltStorage, localPath string) (*importLocalResult, error) { + return importFromLocalJSONLWithOpts(ctx, store, localPath, false) +} + +// importFromLocalJSONLConflictSkip is the auto-import upgrade-recovery +// fallback (GH#3955; the fallbackImporter seam in auto_import_upgrade.go). +// It is identical to importFromLocalJSONLFull except that an issue whose ID +// already exists is left untouched instead of being overwritten, so a +// regressed emptiness guard can never clobber live rows — worst case is a +// no-op. +func importFromLocalJSONLConflictSkip(ctx context.Context, store storage.DoltStorage, localPath string) (*importLocalResult, error) { + return importFromLocalJSONLWithOpts(ctx, store, localPath, true) +} + +// importFromLocalJSONLWithOpts is the shared implementation. It detects +// memory records (lines with "_type":"memory") and imports them via +// SetConfig, while routing regular issue records through the normal path. +// conflictSkip selects insert-if-new (true) vs UPSERT (false) for issue rows. +func importFromLocalJSONLWithOpts(ctx context.Context, store storage.DoltStorage, localPath string, conflictSkip bool) (*importLocalResult, error) { issues, configEntries, err := parseJSONLFile(localPath) if err != nil { return nil, err @@ -285,6 +309,7 @@ func importFromLocalJSONLFull(ctx context.Context, store storage.DoltStorage, lo opts := ImportOptions{ SkipPrefixValidation: true, + ConflictSkip: conflictSkip, } importResult, err := importIssuesCore(ctx, "", store, issues, opts) if err != nil { diff --git a/internal/storage/batch.go b/internal/storage/batch.go index 4108ec431..7861a3c22 100644 --- a/internal/storage/batch.go +++ b/internal/storage/batch.go @@ -22,6 +22,13 @@ type BatchCreateOptions struct { OrphanHandling OrphanHandling // SkipPrefixValidation skips prefix validation for existing IDs (used during import) SkipPrefixValidation bool + // ConflictSkip makes batch creation insert-if-new instead of UPSERT: an + // issue whose ID already exists is left untouched rather than overwritten. + // Used only by the auto-import upgrade-recovery fallback (GH#3955), so + // that if the emptiness guard in maybeAutoImportJSONL ever regresses + // again (cf. PR #3630), auto-import degrades to a harmless no-op instead + // of clobbering live rows. Explicit `bd import` keeps UPSERT semantics. + ConflictSkip bool // SkipDependencyValidationErrors skips dependency validation failures that // legacy imports tolerated, such as cycles or self-dependencies. SkipDependencyValidationErrors bool diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index e5d4f7331..a23c4ab37 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -424,6 +424,12 @@ func (s *EmbeddedDoltStore) ImportJSONLData( if err := issueops.CreateIssuesInTx(ctx, tx, issues, actor, storage.BatchCreateOptions{ OrphanHandling: storage.OrphanAllow, SkipPrefixValidation: true, + // Defense-in-depth (GH#3955): the embedded fast-path is the primary + // auto-import route for 1.0+ users and is gated by the in-transaction + // emptiness check above. Make it insert-if-new too so a regression in + // that check cannot clobber live rows — matching the server-mode + // fallback's conflict-skip behavior. + ConflictSkip: true, }); err != nil { return err } diff --git a/internal/storage/issueops/create.go b/internal/storage/issueops/create.go index a754c64a7..103c560e3 100644 --- a/internal/storage/issueops/create.go +++ b/internal/storage/issueops/create.go @@ -111,7 +111,7 @@ func CreateIssueInTxWithResult(ctx context.Context, tx *sql.Tx, bc *BatchContext return result, nil } - isNew, err := InsertIssueIfNew(ctx, tx, issueTable, issue) + isNew, err := InsertIssueIfNew(ctx, tx, issueTable, issue, bc.Opts.ConflictSkip) if err != nil { return result, err } @@ -495,14 +495,25 @@ func CheckOrphan(ctx context.Context, tx *sql.Tx, issue *types.Issue, issueTable // InsertIssueIfNew inserts the issue and returns whether it was genuinely new. // +// When conflictSkip is true and an issue with the same ID already exists, the +// row is left untouched (no UPSERT) and isNew is false. This is the +// auto-import upgrade-recovery guarantee (GH#3955): even if the emptiness +// guard in maybeAutoImportJSONL regresses, a stale issues.jsonl can never +// overwrite live rows — worst case is a no-op. With conflictSkip false the +// behavior is unchanged: InsertIssueIntoTable runs its INSERT … ON DUPLICATE +// KEY UPDATE, so explicit `bd import` keeps UPSERT semantics. +// //nolint:gosec // G201: table is a hardcoded constant -func InsertIssueIfNew(ctx context.Context, tx *sql.Tx, issueTable string, issue *types.Issue) (isNew bool, err error) { +func InsertIssueIfNew(ctx context.Context, tx *sql.Tx, issueTable string, issue *types.Issue, conflictSkip bool) (isNew bool, err error) { var existingCount int if issue.ID != "" { if err := tx.QueryRowContext(ctx, fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE id = ?`, issueTable), issue.ID).Scan(&existingCount); err != nil { return false, fmt.Errorf("failed to check issue existence for %s: %w", issue.ID, err) } } + if conflictSkip && existingCount > 0 { + return false, nil // issue already exists — skip, never overwrite + } if err := InsertIssueIntoTable(ctx, tx, issueTable, issue); err != nil { return false, fmt.Errorf("failed to insert issue %s: %w", issue.ID, err) } diff --git a/internal/storage/issueops/promote.go b/internal/storage/issueops/promote.go index 8aaed2f68..e82373ec7 100644 --- a/internal/storage/issueops/promote.go +++ b/internal/storage/issueops/promote.go @@ -33,7 +33,7 @@ func PromoteFromEphemeralInTx(ctx context.Context, tx *sql.Tx, id string, actor if err := PrepareIssueForInsert(issue, bc.CustomStatuses, bc.CustomTypes); err != nil { return fmt.Errorf("promote wisp to issues: %w", err) } - if _, err := InsertIssueIfNew(ctx, tx, "issues", issue); err != nil { + if _, err := InsertIssueIfNew(ctx, tx, "issues", issue, false); err != nil { return fmt.Errorf("promote wisp to issues: %w", err) }