Merge pull request #3634 from coffeegoddd/db/atomic-export

atomicfile: fix JSONL export concurrency safety
This commit is contained in:
Dustin Brown 2026-05-01 17:42:24 -07:00 • committed by GitHub
commit 1781ed422e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 423 additions and 18 deletions

View file

@ -10,6 +10,7 @@ import (
"time"
"github.com/spf13/cobra"
"github.com/steveyegge/beads/internal/atomicfile"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/types"
)
@ -63,15 +64,22 @@ func init() {
func runExport(cmd *cobra.Command, args []string) error {
ctx := rootCtx
// Determine output destination
// Determine output destination. File output uses atomic writes
// (temp file + rename) so concurrent exports and crashes never
// leave a truncated or interleaved JSONL file.
var w io.Writer
var aw *atomicfile.Writer
if exportOutput != "" {
f, err := os.Create(exportOutput) //nolint:gosec // user-provided output path
var err error
aw, err = atomicfile.Create(exportOutput, 0o644)
if err != nil {
return fmt.Errorf("failed to create output file: %w", err)
}
defer f.Close()
w = f
defer func() {
// Abort is a no-op if Close was already called.
_ = aw.Abort()
}()
w = aw
} else {
w = os.Stdout
}
@ -218,10 +226,10 @@ func runExport(cmd *cobra.Command, args []string) error {
}
}
// Sync to disk if writing to file
if f, ok := w.(*os.File); ok && f != os.Stdout {
if err := f.Sync(); err != nil {
return fmt.Errorf("failed to sync output file: %w", err)
// Finalize atomic write if writing to file (fsync + rename).
if aw != nil {
if err := aw.Close(); err != nil {
return fmt.Errorf("failed to finalize export file: %w", err)
}
}

View file

@ -11,6 +11,7 @@ import (
"strings"
"time"
"github.com/steveyegge/beads/internal/atomicfile"
"github.com/steveyegge/beads/internal/beads"
"github.com/steveyegge/beads/internal/config"
"github.com/steveyegge/beads/internal/debug"
@ -125,14 +126,19 @@ func maybeAutoExport(ctx context.Context) {
saveExportAutoState(beadsDir, &newState)
}
// exportToFile exports issues + memories to the given file path.
// Used by both `bd export -o` and auto-export.
// exportToFile atomically exports issues + memories to the given file path.
// Writes to a temp file first, then renames into place so readers never see
// a partial or truncated export. Used by both `bd export -o` and auto-export.
func exportToFile(ctx context.Context, path string, includeMemories bool) (issueCount, memoryCount int, err error) {
f, err := os.Create(path) //nolint:gosec // user-configured output path
w, err := atomicfile.Create(path, 0o644)
if err != nil {
return 0, 0, fmt.Errorf("failed to create export file: %w", err)
}
defer f.Close()
defer func() {
if err != nil {
_ = w.Abort()
}
}()
// Build filter: exclude infra types and templates
filter := types.IssueFilter{Limit: 0}
@ -180,7 +186,7 @@ func exportToFile(ctx context.Context, path string, includeMemories bool) (issue
}
// Write issues
enc := json.NewEncoder(f)
enc := json.NewEncoder(w)
for _, issue := range issues {
counts := depCounts[issue.ID]
if counts == nil {
@ -228,10 +234,10 @@ func exportToFile(ctx context.Context, path string, includeMemories bool) (issue
if err != nil {
return issueCount, memoryCount, fmt.Errorf("failed to marshal memory %s: %w", userKey, err)
}
if _, err := f.Write(data); err != nil {
if _, err := w.Write(data); err != nil {
return issueCount, memoryCount, fmt.Errorf("failed to write memory: %w", err)
}
if _, err := f.Write([]byte{'\n'}); err != nil {
if _, err := w.Write([]byte{'\n'}); err != nil {
return issueCount, memoryCount, fmt.Errorf("failed to write newline: %w", err)
}
memoryCount++
@ -239,8 +245,8 @@ func exportToFile(ctx context.Context, path string, includeMemories bool) (issue
}
}
if err := f.Sync(); err != nil {
return issueCount, memoryCount, fmt.Errorf("failed to sync: %w", err)
if err := w.Close(); err != nil {
return issueCount, memoryCount, fmt.Errorf("failed to finalize export: %w", err)
}
return issueCount, memoryCount, nil
@ -266,7 +272,7 @@ func saveExportAutoState(beadsDir string, state *exportAutoState) {
fmt.Fprintf(os.Stderr, "Warning: auto-export: failed to marshal state: %v\n", err)
return
}
if err := os.WriteFile(path, data, 0o600); err != nil {
if err := atomicfile.WriteFile(path, data, 0o600); err != nil {
fmt.Fprintf(os.Stderr, "Warning: auto-export: failed to save state: %v\n", err)
}
}

View file

@ -0,0 +1,110 @@
// Package atomicfile provides atomic file writes via temp-file + rename.
//
// Writes land in a temporary file in the same directory as the target,
// are fsynced, then atomically renamed into place. Readers never see a
// partial or truncated file — only the previous complete version or the
// new complete version.
package atomicfile
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
)
// WriteFile atomically writes data to path with the given permissions.
// It creates an atomic Writer, copies data in via io.Copy, then
// fsyncs and renames into place.
func WriteFile(path string, data []byte, perm os.FileMode) error {
w, err := Create(path, perm)
if err != nil {
return err
}
if _, err := io.Copy(w, bytes.NewReader(data)); err != nil {
_ = w.Abort()
return err
}
return w.Close()
}
// Writer is an io.WriteCloser that writes to a temporary file and
// atomically renames it to the target path on Close. Call Abort to
// discard the temp file without touching the target.
type Writer struct {
target string
f *os.File
perm os.FileMode
done bool
}
// Create returns a Writer that will atomically replace path on Close.
// The temp file is created in the same directory as path to guarantee
// same-filesystem rename semantics.
func Create(path string, perm os.FileMode) (*Writer, error) {
dir := filepath.Dir(path)
base := filepath.Base(path)
f, err := os.CreateTemp(dir, ".~"+base+".")
if err != nil {
return nil, fmt.Errorf("atomicfile: create temp: %w", err)
}
return &Writer{
target: path,
f: f,
perm: perm,
}, nil
}
// Write delegates to the underlying temp file.
func (w *Writer) Write(p []byte) (int, error) {
return w.f.Write(p)
}
// Close fsyncs the temp file and atomically renames it to the target path.
// After Close returns successfully, the target contains exactly the data
// written. On error the temp file is removed and the target is untouched.
func (w *Writer) Close() error {
if w.done {
return nil
}
w.done = true
// Ensure permissions before rename — CreateTemp uses 0600 by default.
if err := w.f.Chmod(w.perm); err != nil {
_ = w.f.Close()
_ = os.Remove(w.f.Name())
return fmt.Errorf("atomicfile: chmod: %w", err)
}
if err := w.f.Sync(); err != nil {
_ = w.f.Close()
_ = os.Remove(w.f.Name())
return fmt.Errorf("atomicfile: sync: %w", err)
}
if err := w.f.Close(); err != nil {
_ = os.Remove(w.f.Name())
return fmt.Errorf("atomicfile: close: %w", err)
}
if err := os.Rename(w.f.Name(), w.target); err != nil {
_ = os.Remove(w.f.Name())
return fmt.Errorf("atomicfile: rename: %w", err)
}
return nil
}
// Abort discards the temp file without renaming. The target is untouched.
// Safe to call multiple times or after Close.
func (w *Writer) Abort() error {
if w.done {
return nil
}
w.done = true
_ = w.f.Close()
return os.Remove(w.f.Name())
}

View file

@ -0,0 +1,281 @@
package atomicfile
import (
"os"
"path/filepath"
"strings"
"sync"
"testing"
)
func TestWriteFile_Basic(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "out.txt")
if err := WriteFile(path, []byte("hello"), 0o644); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(got) != "hello" {
t.Errorf("got %q, want %q", got, "hello")
}
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if perm := info.Mode().Perm(); perm != 0o644 {
t.Errorf("perm = %o, want 0644", perm)
}
}
func TestWriteFile_Overwrite(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "out.txt")
if err := os.WriteFile(path, []byte("original"), 0o644); err != nil {
t.Fatal(err)
}
if err := WriteFile(path, []byte("replaced"), 0o644); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(got) != "replaced" {
t.Errorf("got %q, want %q", got, "replaced")
}
}
func TestCreate_Close(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "streamed.txt")
w, err := Create(path, 0o644)
if err != nil {
t.Fatal(err)
}
for _, chunk := range []string{"line1\n", "line2\n", "line3\n"} {
if _, err := w.Write([]byte(chunk)); err != nil {
t.Fatal(err)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
want := "line1\nline2\nline3\n"
if string(got) != want {
t.Errorf("got %q, want %q", got, want)
}
}
func TestCreate_Abort(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "aborted.txt")
w, err := Create(path, 0o644)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write([]byte("should not appear")); err != nil {
t.Fatal(err)
}
if err := w.Abort(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Errorf("expected target to not exist after Abort, got err=%v", err)
}
}
func TestCreate_Abort_PreservesExisting(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "existing.txt")
if err := os.WriteFile(path, []byte("keep me"), 0o644); err != nil {
t.Fatal(err)
}
w, err := Create(path, 0o644)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write([]byte("overwrite attempt")); err != nil {
t.Fatal(err)
}
if err := w.Abort(); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(got) != "keep me" {
t.Errorf("original content clobbered: got %q, want %q", got, "keep me")
}
}
func TestWriteFile_TempCleanup(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "clean.txt")
if err := WriteFile(path, []byte("data"), 0o644); err != nil {
t.Fatal(err)
}
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
if strings.HasPrefix(e.Name(), ".~") {
t.Errorf("temp file left behind: %s", e.Name())
}
}
}
func TestWriteFile_SameDirectory(t *testing.T) {
t.Parallel()
dir := t.TempDir()
sub := filepath.Join(dir, "sub")
if err := os.MkdirAll(sub, 0o755); err != nil {
t.Fatal(err)
}
path := filepath.Join(sub, "target.txt")
// Create and immediately abort so we can inspect the temp file location.
w, err := Create(path, 0o644)
if err != nil {
t.Fatal(err)
}
tmpDir := filepath.Dir(w.f.Name())
_ = w.Abort()
if tmpDir != sub {
t.Errorf("temp file in %q, want %q (same directory as target)", tmpDir, sub)
}
}
func TestConcurrentWriters(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "concurrent.txt")
const numWriters = 20
const dataSize = 4096
var wg sync.WaitGroup
wg.Add(numWriters)
for i := 0; i < numWriters; i++ {
go func(id int) {
defer wg.Done()
// Each writer writes a distinct byte repeated dataSize times.
data := make([]byte, dataSize)
for j := range data {
data[j] = byte('A' + id%26)
}
// Errors from concurrent rename races are acceptable;
// the point is the final file must be valid.
_ = WriteFile(path, data, 0o644)
}(i)
}
wg.Wait()
got, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if len(got) != dataSize {
t.Fatalf("file size = %d, want %d", len(got), dataSize)
}
// Every byte must be the same character — no interleaving from
// different writers.
first := got[0]
for i, b := range got {
if b != first {
t.Fatalf("corruption at byte %d: got %c, expected %c (consistent single-writer content)", i, b, first)
}
}
}
func TestConcurrentWriters_NoCorruption(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "nocorrupt.jsonl")
const numWriters = 20
// Simulate JSONL export: each writer writes multiple lines.
var wg sync.WaitGroup
wg.Add(numWriters)
for i := 0; i < numWriters; i++ {
go func(id int) {
defer wg.Done()
w, err := Create(path, 0o644)
if err != nil {
return // concurrent temp file creation can race; ok
}
for line := 0; line < 10; line++ {
data := []byte(strings.Repeat(string(rune('A'+id%26)), 80) + "\n")
if _, err := w.Write(data); err != nil {
_ = w.Abort()
return
}
}
// Close may fail if another writer renamed over us; that's fine.
if err := w.Close(); err != nil {
return
}
}(i)
}
wg.Wait()
got, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
lines := strings.Split(strings.TrimSuffix(string(got), "\n"), "\n")
if len(lines) != 10 {
t.Fatalf("expected 10 lines, got %d", len(lines))
}
// All lines must contain the same character — proving the file came
// from a single writer, not interleaved from multiple.
firstChar := lines[0][0]
for i, line := range lines {
if len(line) != 80 {
t.Fatalf("line %d length = %d, want 80", i, len(line))
}
for j, b := range []byte(line) {
if b != firstChar {
t.Fatalf("line %d byte %d: got %c, expected %c (interleaved writers)", i, j, b, firstChar)
}
}
}
}