diff --git a/cmd/bd/export.go b/cmd/bd/export.go index 48559433b..6be284f10 100644 --- a/cmd/bd/export.go +++ b/cmd/bd/export.go @@ -10,6 +10,7 @@ import ( "time" "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/atomicfile" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" ) @@ -63,15 +64,22 @@ func init() { func runExport(cmd *cobra.Command, args []string) error { ctx := rootCtx - // Determine output destination + // Determine output destination. File output uses atomic writes + // (temp file + rename) so concurrent exports and crashes never + // leave a truncated or interleaved JSONL file. var w io.Writer + var aw *atomicfile.Writer if exportOutput != "" { - f, err := os.Create(exportOutput) //nolint:gosec // user-provided output path + var err error + aw, err = atomicfile.Create(exportOutput, 0o644) if err != nil { return fmt.Errorf("failed to create output file: %w", err) } - defer f.Close() - w = f + defer func() { + // Abort is a no-op if Close was already called. + _ = aw.Abort() + }() + w = aw } else { w = os.Stdout } @@ -218,10 +226,10 @@ func runExport(cmd *cobra.Command, args []string) error { } } - // Sync to disk if writing to file - if f, ok := w.(*os.File); ok && f != os.Stdout { - if err := f.Sync(); err != nil { - return fmt.Errorf("failed to sync output file: %w", err) + // Finalize atomic write if writing to file (fsync + rename). + if aw != nil { + if err := aw.Close(); err != nil { + return fmt.Errorf("failed to finalize export file: %w", err) } } diff --git a/cmd/bd/export_auto.go b/cmd/bd/export_auto.go index c1a6b17ee..65bcbc79f 100644 --- a/cmd/bd/export_auto.go +++ b/cmd/bd/export_auto.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/steveyegge/beads/internal/atomicfile" "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/debug" @@ -125,14 +126,19 @@ func maybeAutoExport(ctx context.Context) { saveExportAutoState(beadsDir, &newState) } -// exportToFile exports issues + memories to the given file path. -// Used by both `bd export -o` and auto-export. +// exportToFile atomically exports issues + memories to the given file path. +// Writes to a temp file first, then renames into place so readers never see +// a partial or truncated export. Used by both `bd export -o` and auto-export. func exportToFile(ctx context.Context, path string, includeMemories bool) (issueCount, memoryCount int, err error) { - f, err := os.Create(path) //nolint:gosec // user-configured output path + w, err := atomicfile.Create(path, 0o644) if err != nil { return 0, 0, fmt.Errorf("failed to create export file: %w", err) } - defer f.Close() + defer func() { + if err != nil { + _ = w.Abort() + } + }() // Build filter: exclude infra types and templates filter := types.IssueFilter{Limit: 0} @@ -180,7 +186,7 @@ func exportToFile(ctx context.Context, path string, includeMemories bool) (issue } // Write issues - enc := json.NewEncoder(f) + enc := json.NewEncoder(w) for _, issue := range issues { counts := depCounts[issue.ID] if counts == nil { @@ -228,10 +234,10 @@ func exportToFile(ctx context.Context, path string, includeMemories bool) (issue if err != nil { return issueCount, memoryCount, fmt.Errorf("failed to marshal memory %s: %w", userKey, err) } - if _, err := f.Write(data); err != nil { + if _, err := w.Write(data); err != nil { return issueCount, memoryCount, fmt.Errorf("failed to write memory: %w", err) } - if _, err := f.Write([]byte{'\n'}); err != nil { + if _, err := w.Write([]byte{'\n'}); err != nil { return issueCount, memoryCount, fmt.Errorf("failed to write newline: %w", err) } memoryCount++ @@ -239,8 +245,8 @@ func exportToFile(ctx context.Context, path string, includeMemories bool) (issue } } - if err := f.Sync(); err != nil { - return issueCount, memoryCount, fmt.Errorf("failed to sync: %w", err) + if err := w.Close(); err != nil { + return issueCount, memoryCount, fmt.Errorf("failed to finalize export: %w", err) } return issueCount, memoryCount, nil @@ -266,7 +272,7 @@ func saveExportAutoState(beadsDir string, state *exportAutoState) { fmt.Fprintf(os.Stderr, "Warning: auto-export: failed to marshal state: %v\n", err) return } - if err := os.WriteFile(path, data, 0o600); err != nil { + if err := atomicfile.WriteFile(path, data, 0o600); err != nil { fmt.Fprintf(os.Stderr, "Warning: auto-export: failed to save state: %v\n", err) } } diff --git a/internal/atomicfile/atomicfile.go b/internal/atomicfile/atomicfile.go new file mode 100644 index 000000000..c9a147862 --- /dev/null +++ b/internal/atomicfile/atomicfile.go @@ -0,0 +1,110 @@ +// Package atomicfile provides atomic file writes via temp-file + rename. +// +// Writes land in a temporary file in the same directory as the target, +// are fsynced, then atomically renamed into place. Readers never see a +// partial or truncated file — only the previous complete version or the +// new complete version. +package atomicfile + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" +) + +// WriteFile atomically writes data to path with the given permissions. +// It creates an atomic Writer, copies data in via io.Copy, then +// fsyncs and renames into place. +func WriteFile(path string, data []byte, perm os.FileMode) error { + w, err := Create(path, perm) + if err != nil { + return err + } + if _, err := io.Copy(w, bytes.NewReader(data)); err != nil { + _ = w.Abort() + return err + } + return w.Close() +} + +// Writer is an io.WriteCloser that writes to a temporary file and +// atomically renames it to the target path on Close. Call Abort to +// discard the temp file without touching the target. +type Writer struct { + target string + f *os.File + perm os.FileMode + done bool +} + +// Create returns a Writer that will atomically replace path on Close. +// The temp file is created in the same directory as path to guarantee +// same-filesystem rename semantics. +func Create(path string, perm os.FileMode) (*Writer, error) { + dir := filepath.Dir(path) + base := filepath.Base(path) + + f, err := os.CreateTemp(dir, ".~"+base+".") + if err != nil { + return nil, fmt.Errorf("atomicfile: create temp: %w", err) + } + + return &Writer{ + target: path, + f: f, + perm: perm, + }, nil +} + +// Write delegates to the underlying temp file. +func (w *Writer) Write(p []byte) (int, error) { + return w.f.Write(p) +} + +// Close fsyncs the temp file and atomically renames it to the target path. +// After Close returns successfully, the target contains exactly the data +// written. On error the temp file is removed and the target is untouched. +func (w *Writer) Close() error { + if w.done { + return nil + } + w.done = true + + // Ensure permissions before rename — CreateTemp uses 0600 by default. + if err := w.f.Chmod(w.perm); err != nil { + _ = w.f.Close() + _ = os.Remove(w.f.Name()) + return fmt.Errorf("atomicfile: chmod: %w", err) + } + + if err := w.f.Sync(); err != nil { + _ = w.f.Close() + _ = os.Remove(w.f.Name()) + return fmt.Errorf("atomicfile: sync: %w", err) + } + + if err := w.f.Close(); err != nil { + _ = os.Remove(w.f.Name()) + return fmt.Errorf("atomicfile: close: %w", err) + } + + if err := os.Rename(w.f.Name(), w.target); err != nil { + _ = os.Remove(w.f.Name()) + return fmt.Errorf("atomicfile: rename: %w", err) + } + + return nil +} + +// Abort discards the temp file without renaming. The target is untouched. +// Safe to call multiple times or after Close. +func (w *Writer) Abort() error { + if w.done { + return nil + } + w.done = true + _ = w.f.Close() + return os.Remove(w.f.Name()) +} diff --git a/internal/atomicfile/atomicfile_test.go b/internal/atomicfile/atomicfile_test.go new file mode 100644 index 000000000..e33b382cd --- /dev/null +++ b/internal/atomicfile/atomicfile_test.go @@ -0,0 +1,281 @@ +package atomicfile + +import ( + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +func TestWriteFile_Basic(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "out.txt") + + if err := WriteFile(path, []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != "hello" { + t.Errorf("got %q, want %q", got, "hello") + } + + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if perm := info.Mode().Perm(); perm != 0o644 { + t.Errorf("perm = %o, want 0644", perm) + } +} + +func TestWriteFile_Overwrite(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "out.txt") + + if err := os.WriteFile(path, []byte("original"), 0o644); err != nil { + t.Fatal(err) + } + + if err := WriteFile(path, []byte("replaced"), 0o644); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != "replaced" { + t.Errorf("got %q, want %q", got, "replaced") + } +} + +func TestCreate_Close(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "streamed.txt") + + w, err := Create(path, 0o644) + if err != nil { + t.Fatal(err) + } + + for _, chunk := range []string{"line1\n", "line2\n", "line3\n"} { + if _, err := w.Write([]byte(chunk)); err != nil { + t.Fatal(err) + } + } + + if err := w.Close(); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + want := "line1\nline2\nline3\n" + if string(got) != want { + t.Errorf("got %q, want %q", got, want) + } +} + +func TestCreate_Abort(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "aborted.txt") + + w, err := Create(path, 0o644) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte("should not appear")); err != nil { + t.Fatal(err) + } + if err := w.Abort(); err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("expected target to not exist after Abort, got err=%v", err) + } +} + +func TestCreate_Abort_PreservesExisting(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "existing.txt") + + if err := os.WriteFile(path, []byte("keep me"), 0o644); err != nil { + t.Fatal(err) + } + + w, err := Create(path, 0o644) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte("overwrite attempt")); err != nil { + t.Fatal(err) + } + if err := w.Abort(); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != "keep me" { + t.Errorf("original content clobbered: got %q, want %q", got, "keep me") + } +} + +func TestWriteFile_TempCleanup(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "clean.txt") + + if err := WriteFile(path, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".~") { + t.Errorf("temp file left behind: %s", e.Name()) + } + } +} + +func TestWriteFile_SameDirectory(t *testing.T) { + t.Parallel() + dir := t.TempDir() + sub := filepath.Join(dir, "sub") + if err := os.MkdirAll(sub, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(sub, "target.txt") + + // Create and immediately abort so we can inspect the temp file location. + w, err := Create(path, 0o644) + if err != nil { + t.Fatal(err) + } + tmpDir := filepath.Dir(w.f.Name()) + _ = w.Abort() + + if tmpDir != sub { + t.Errorf("temp file in %q, want %q (same directory as target)", tmpDir, sub) + } +} + +func TestConcurrentWriters(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "concurrent.txt") + + const numWriters = 20 + const dataSize = 4096 + + var wg sync.WaitGroup + wg.Add(numWriters) + + for i := 0; i < numWriters; i++ { + go func(id int) { + defer wg.Done() + // Each writer writes a distinct byte repeated dataSize times. + data := make([]byte, dataSize) + for j := range data { + data[j] = byte('A' + id%26) + } + // Errors from concurrent rename races are acceptable; + // the point is the final file must be valid. + _ = WriteFile(path, data, 0o644) + }(i) + } + wg.Wait() + + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if len(got) != dataSize { + t.Fatalf("file size = %d, want %d", len(got), dataSize) + } + + // Every byte must be the same character — no interleaving from + // different writers. + first := got[0] + for i, b := range got { + if b != first { + t.Fatalf("corruption at byte %d: got %c, expected %c (consistent single-writer content)", i, b, first) + } + } +} + +func TestConcurrentWriters_NoCorruption(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "nocorrupt.jsonl") + + const numWriters = 20 + + // Simulate JSONL export: each writer writes multiple lines. + var wg sync.WaitGroup + wg.Add(numWriters) + + for i := 0; i < numWriters; i++ { + go func(id int) { + defer wg.Done() + w, err := Create(path, 0o644) + if err != nil { + return // concurrent temp file creation can race; ok + } + for line := 0; line < 10; line++ { + data := []byte(strings.Repeat(string(rune('A'+id%26)), 80) + "\n") + if _, err := w.Write(data); err != nil { + _ = w.Abort() + return + } + } + // Close may fail if another writer renamed over us; that's fine. + if err := w.Close(); err != nil { + return + } + }(i) + } + wg.Wait() + + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + lines := strings.Split(strings.TrimSuffix(string(got), "\n"), "\n") + if len(lines) != 10 { + t.Fatalf("expected 10 lines, got %d", len(lines)) + } + + // All lines must contain the same character — proving the file came + // from a single writer, not interleaved from multiple. + firstChar := lines[0][0] + for i, line := range lines { + if len(line) != 80 { + t.Fatalf("line %d length = %d, want 80", i, len(line)) + } + for j, b := range []byte(line) { + if b != firstChar { + t.Fatalf("line %d byte %d: got %c, expected %c (interleaved writers)", i, j, b, firstChar) + } + } + } +}