Merge pull request #3634 from coffeegoddd/db/atomic-export
atomicfile: fix JSONL export concurrency safety
This commit is contained in:
commit
1781ed422e
4 changed files with 423 additions and 18 deletions
|
|
@ -10,6 +10,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/steveyegge/beads/internal/atomicfile"
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
|
@ -63,15 +64,22 @@ func init() {
|
|||
func runExport(cmd *cobra.Command, args []string) error {
|
||||
ctx := rootCtx
|
||||
|
||||
// Determine output destination
|
||||
// Determine output destination. File output uses atomic writes
|
||||
// (temp file + rename) so concurrent exports and crashes never
|
||||
// leave a truncated or interleaved JSONL file.
|
||||
var w io.Writer
|
||||
var aw *atomicfile.Writer
|
||||
if exportOutput != "" {
|
||||
f, err := os.Create(exportOutput) //nolint:gosec // user-provided output path
|
||||
var err error
|
||||
aw, err = atomicfile.Create(exportOutput, 0o644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create output file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
w = f
|
||||
defer func() {
|
||||
// Abort is a no-op if Close was already called.
|
||||
_ = aw.Abort()
|
||||
}()
|
||||
w = aw
|
||||
} else {
|
||||
w = os.Stdout
|
||||
}
|
||||
|
|
@ -218,10 +226,10 @@ func runExport(cmd *cobra.Command, args []string) error {
|
|||
}
|
||||
}
|
||||
|
||||
// Sync to disk if writing to file
|
||||
if f, ok := w.(*os.File); ok && f != os.Stdout {
|
||||
if err := f.Sync(); err != nil {
|
||||
return fmt.Errorf("failed to sync output file: %w", err)
|
||||
// Finalize atomic write if writing to file (fsync + rename).
|
||||
if aw != nil {
|
||||
if err := aw.Close(); err != nil {
|
||||
return fmt.Errorf("failed to finalize export file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/atomicfile"
|
||||
"github.com/steveyegge/beads/internal/beads"
|
||||
"github.com/steveyegge/beads/internal/config"
|
||||
"github.com/steveyegge/beads/internal/debug"
|
||||
|
|
@ -125,14 +126,19 @@ func maybeAutoExport(ctx context.Context) {
|
|||
saveExportAutoState(beadsDir, &newState)
|
||||
}
|
||||
|
||||
// exportToFile exports issues + memories to the given file path.
|
||||
// Used by both `bd export -o` and auto-export.
|
||||
// exportToFile atomically exports issues + memories to the given file path.
|
||||
// Writes to a temp file first, then renames into place so readers never see
|
||||
// a partial or truncated export. Used by both `bd export -o` and auto-export.
|
||||
func exportToFile(ctx context.Context, path string, includeMemories bool) (issueCount, memoryCount int, err error) {
|
||||
f, err := os.Create(path) //nolint:gosec // user-configured output path
|
||||
w, err := atomicfile.Create(path, 0o644)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to create export file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
defer func() {
|
||||
if err != nil {
|
||||
_ = w.Abort()
|
||||
}
|
||||
}()
|
||||
|
||||
// Build filter: exclude infra types and templates
|
||||
filter := types.IssueFilter{Limit: 0}
|
||||
|
|
@ -180,7 +186,7 @@ func exportToFile(ctx context.Context, path string, includeMemories bool) (issue
|
|||
}
|
||||
|
||||
// Write issues
|
||||
enc := json.NewEncoder(f)
|
||||
enc := json.NewEncoder(w)
|
||||
for _, issue := range issues {
|
||||
counts := depCounts[issue.ID]
|
||||
if counts == nil {
|
||||
|
|
@ -228,10 +234,10 @@ func exportToFile(ctx context.Context, path string, includeMemories bool) (issue
|
|||
if err != nil {
|
||||
return issueCount, memoryCount, fmt.Errorf("failed to marshal memory %s: %w", userKey, err)
|
||||
}
|
||||
if _, err := f.Write(data); err != nil {
|
||||
if _, err := w.Write(data); err != nil {
|
||||
return issueCount, memoryCount, fmt.Errorf("failed to write memory: %w", err)
|
||||
}
|
||||
if _, err := f.Write([]byte{'\n'}); err != nil {
|
||||
if _, err := w.Write([]byte{'\n'}); err != nil {
|
||||
return issueCount, memoryCount, fmt.Errorf("failed to write newline: %w", err)
|
||||
}
|
||||
memoryCount++
|
||||
|
|
@ -239,8 +245,8 @@ func exportToFile(ctx context.Context, path string, includeMemories bool) (issue
|
|||
}
|
||||
}
|
||||
|
||||
if err := f.Sync(); err != nil {
|
||||
return issueCount, memoryCount, fmt.Errorf("failed to sync: %w", err)
|
||||
if err := w.Close(); err != nil {
|
||||
return issueCount, memoryCount, fmt.Errorf("failed to finalize export: %w", err)
|
||||
}
|
||||
|
||||
return issueCount, memoryCount, nil
|
||||
|
|
@ -266,7 +272,7 @@ func saveExportAutoState(beadsDir string, state *exportAutoState) {
|
|||
fmt.Fprintf(os.Stderr, "Warning: auto-export: failed to marshal state: %v\n", err)
|
||||
return
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o600); err != nil {
|
||||
if err := atomicfile.WriteFile(path, data, 0o600); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Warning: auto-export: failed to save state: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
110
internal/atomicfile/atomicfile.go
Normal file
110
internal/atomicfile/atomicfile.go
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
// Package atomicfile provides atomic file writes via temp-file + rename.
|
||||
//
|
||||
// Writes land in a temporary file in the same directory as the target,
|
||||
// are fsynced, then atomically renamed into place. Readers never see a
|
||||
// partial or truncated file — only the previous complete version or the
|
||||
// new complete version.
|
||||
package atomicfile
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// WriteFile atomically writes data to path with the given permissions.
|
||||
// It creates an atomic Writer, copies data in via io.Copy, then
|
||||
// fsyncs and renames into place.
|
||||
func WriteFile(path string, data []byte, perm os.FileMode) error {
|
||||
w, err := Create(path, perm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(w, bytes.NewReader(data)); err != nil {
|
||||
_ = w.Abort()
|
||||
return err
|
||||
}
|
||||
return w.Close()
|
||||
}
|
||||
|
||||
// Writer is an io.WriteCloser that writes to a temporary file and
|
||||
// atomically renames it to the target path on Close. Call Abort to
|
||||
// discard the temp file without touching the target.
|
||||
type Writer struct {
|
||||
target string
|
||||
f *os.File
|
||||
perm os.FileMode
|
||||
done bool
|
||||
}
|
||||
|
||||
// Create returns a Writer that will atomically replace path on Close.
|
||||
// The temp file is created in the same directory as path to guarantee
|
||||
// same-filesystem rename semantics.
|
||||
func Create(path string, perm os.FileMode) (*Writer, error) {
|
||||
dir := filepath.Dir(path)
|
||||
base := filepath.Base(path)
|
||||
|
||||
f, err := os.CreateTemp(dir, ".~"+base+".")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("atomicfile: create temp: %w", err)
|
||||
}
|
||||
|
||||
return &Writer{
|
||||
target: path,
|
||||
f: f,
|
||||
perm: perm,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Write delegates to the underlying temp file.
|
||||
func (w *Writer) Write(p []byte) (int, error) {
|
||||
return w.f.Write(p)
|
||||
}
|
||||
|
||||
// Close fsyncs the temp file and atomically renames it to the target path.
|
||||
// After Close returns successfully, the target contains exactly the data
|
||||
// written. On error the temp file is removed and the target is untouched.
|
||||
func (w *Writer) Close() error {
|
||||
if w.done {
|
||||
return nil
|
||||
}
|
||||
w.done = true
|
||||
|
||||
// Ensure permissions before rename — CreateTemp uses 0600 by default.
|
||||
if err := w.f.Chmod(w.perm); err != nil {
|
||||
_ = w.f.Close()
|
||||
_ = os.Remove(w.f.Name())
|
||||
return fmt.Errorf("atomicfile: chmod: %w", err)
|
||||
}
|
||||
|
||||
if err := w.f.Sync(); err != nil {
|
||||
_ = w.f.Close()
|
||||
_ = os.Remove(w.f.Name())
|
||||
return fmt.Errorf("atomicfile: sync: %w", err)
|
||||
}
|
||||
|
||||
if err := w.f.Close(); err != nil {
|
||||
_ = os.Remove(w.f.Name())
|
||||
return fmt.Errorf("atomicfile: close: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(w.f.Name(), w.target); err != nil {
|
||||
_ = os.Remove(w.f.Name())
|
||||
return fmt.Errorf("atomicfile: rename: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Abort discards the temp file without renaming. The target is untouched.
|
||||
// Safe to call multiple times or after Close.
|
||||
func (w *Writer) Abort() error {
|
||||
if w.done {
|
||||
return nil
|
||||
}
|
||||
w.done = true
|
||||
_ = w.f.Close()
|
||||
return os.Remove(w.f.Name())
|
||||
}
|
||||
281
internal/atomicfile/atomicfile_test.go
Normal file
281
internal/atomicfile/atomicfile_test.go
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
package atomicfile
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWriteFile_Basic(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "out.txt")
|
||||
|
||||
if err := WriteFile(path, []byte("hello"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "hello" {
|
||||
t.Errorf("got %q, want %q", got, "hello")
|
||||
}
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if perm := info.Mode().Perm(); perm != 0o644 {
|
||||
t.Errorf("perm = %o, want 0644", perm)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFile_Overwrite(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "out.txt")
|
||||
|
||||
if err := os.WriteFile(path, []byte("original"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := WriteFile(path, []byte("replaced"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "replaced" {
|
||||
t.Errorf("got %q, want %q", got, "replaced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_Close(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "streamed.txt")
|
||||
|
||||
w, err := Create(path, 0o644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, chunk := range []string{"line1\n", "line2\n", "line3\n"} {
|
||||
if _, err := w.Write([]byte(chunk)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "line1\nline2\nline3\n"
|
||||
if string(got) != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_Abort(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "aborted.txt")
|
||||
|
||||
w, err := Create(path, 0o644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write([]byte("should not appear")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Abort(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Errorf("expected target to not exist after Abort, got err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreate_Abort_PreservesExisting(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "existing.txt")
|
||||
|
||||
if err := os.WriteFile(path, []byte("keep me"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
w, err := Create(path, 0o644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write([]byte("overwrite attempt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.Abort(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "keep me" {
|
||||
t.Errorf("original content clobbered: got %q, want %q", got, "keep me")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFile_TempCleanup(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "clean.txt")
|
||||
|
||||
if err := WriteFile(path, []byte("data"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if strings.HasPrefix(e.Name(), ".~") {
|
||||
t.Errorf("temp file left behind: %s", e.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFile_SameDirectory(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
sub := filepath.Join(dir, "sub")
|
||||
if err := os.MkdirAll(sub, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(sub, "target.txt")
|
||||
|
||||
// Create and immediately abort so we can inspect the temp file location.
|
||||
w, err := Create(path, 0o644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tmpDir := filepath.Dir(w.f.Name())
|
||||
_ = w.Abort()
|
||||
|
||||
if tmpDir != sub {
|
||||
t.Errorf("temp file in %q, want %q (same directory as target)", tmpDir, sub)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentWriters(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "concurrent.txt")
|
||||
|
||||
const numWriters = 20
|
||||
const dataSize = 4096
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(numWriters)
|
||||
|
||||
for i := 0; i < numWriters; i++ {
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
// Each writer writes a distinct byte repeated dataSize times.
|
||||
data := make([]byte, dataSize)
|
||||
for j := range data {
|
||||
data[j] = byte('A' + id%26)
|
||||
}
|
||||
// Errors from concurrent rename races are acceptable;
|
||||
// the point is the final file must be valid.
|
||||
_ = WriteFile(path, data, 0o644)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != dataSize {
|
||||
t.Fatalf("file size = %d, want %d", len(got), dataSize)
|
||||
}
|
||||
|
||||
// Every byte must be the same character — no interleaving from
|
||||
// different writers.
|
||||
first := got[0]
|
||||
for i, b := range got {
|
||||
if b != first {
|
||||
t.Fatalf("corruption at byte %d: got %c, expected %c (consistent single-writer content)", i, b, first)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentWriters_NoCorruption(t *testing.T) {
|
||||
t.Parallel()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "nocorrupt.jsonl")
|
||||
|
||||
const numWriters = 20
|
||||
|
||||
// Simulate JSONL export: each writer writes multiple lines.
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(numWriters)
|
||||
|
||||
for i := 0; i < numWriters; i++ {
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
w, err := Create(path, 0o644)
|
||||
if err != nil {
|
||||
return // concurrent temp file creation can race; ok
|
||||
}
|
||||
for line := 0; line < 10; line++ {
|
||||
data := []byte(strings.Repeat(string(rune('A'+id%26)), 80) + "\n")
|
||||
if _, err := w.Write(data); err != nil {
|
||||
_ = w.Abort()
|
||||
return
|
||||
}
|
||||
}
|
||||
// Close may fail if another writer renamed over us; that's fine.
|
||||
if err := w.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lines := strings.Split(strings.TrimSuffix(string(got), "\n"), "\n")
|
||||
if len(lines) != 10 {
|
||||
t.Fatalf("expected 10 lines, got %d", len(lines))
|
||||
}
|
||||
|
||||
// All lines must contain the same character — proving the file came
|
||||
// from a single writer, not interleaved from multiple.
|
||||
firstChar := lines[0][0]
|
||||
for i, line := range lines {
|
||||
if len(line) != 80 {
|
||||
t.Fatalf("line %d length = %d, want 80", i, len(line))
|
||||
}
|
||||
for j, b := range []byte(line) {
|
||||
if b != firstChar {
|
||||
t.Fatalf("line %d byte %d: got %c, expected %c (interleaved writers)", i, j, b, firstChar)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue