fix(test): resolve cobra OutOrStdout data race in CI (#1916)
* fix(test): remove t.Parallel() from cobra Help tests to fix data race cobra's OutOrStdout() eagerly evaluates os.Stdout as the default argument even when cmd.SetOut() has been called, so any parallel test calling cmd.Help() or cmd.Execute() races with captureStdout() which redirects os.Stdout behind stdioMutex. Remove t.Parallel() from the three TestSearchCommand_* tests that exercise this path. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * test: add cobra-parallel policy guard and stdio mutex contract test - TestCobraParallelPolicyGuard: scans test files and fails if any parallel test calls cobra Help()/Execute() without SetOut/SetErr. Strips comments to avoid false positives. - TestStdioMutexContract: deterministic reentrancy check proving stdioMutex serializes captureStdout calls (fails without -race). - Update stdioMutex documentation to explain that SetOut is NOT sufficient to prevent the race (cobra eagerly reads os.Stdout). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * refactor(test): strengthen cobra-parallel policy guard Remove the SetOut/SetErr escape hatch since cobra's OutOrStdout() eagerly evaluates os.Stdout even when outWriter is set. The guard now forbids ALL cobra output methods in parallel tests. Also adds stripStringLiterals() to prevent self-matching false positives. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
718a5e2c25
commit
ba29fdc1f7
3 changed files with 199 additions and 5 deletions
|
|
@ -23,8 +23,8 @@ import (
|
|||
// - Even if Help() fails (e.g., output redirection fails), we still exit with code 1
|
||||
// - The error from Help() is rare (typically I/O errors writing to stderr)
|
||||
// - Since we're already in an error state, ignoring Help() errors is acceptable
|
||||
// NOT parallel: see TestSearchCommand_MissingQueryShowsHelp comment (cobra OutOrStdout race).
|
||||
func TestSearchCommand_HelpErrorHandling(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Create a test command similar to searchCmd
|
||||
cmd := &cobra.Command{
|
||||
Use: "search [query]",
|
||||
|
|
@ -77,9 +77,9 @@ func TestSearchCommand_HelpErrorHandling(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
// TestSearchCommand_HelpSuppression verifies that #nosec comment is appropriate
|
||||
// TestSearchCommand_HelpSuppression verifies that #nosec comment is appropriate.
|
||||
// NOT parallel: see TestSearchCommand_MissingQueryShowsHelp comment (cobra OutOrStdout race).
|
||||
func TestSearchCommand_HelpSuppression(t *testing.T) {
|
||||
t.Parallel()
|
||||
// This test documents why ignoring cmd.Help() error is safe:
|
||||
//
|
||||
// 1. Help() is called in an error path (missing required argument)
|
||||
|
|
@ -119,9 +119,11 @@ func (fw *failingWriter) Write(p []byte) (n int, err error) {
|
|||
return 0, io.ErrClosedPipe // Simulate I/O error
|
||||
}
|
||||
|
||||
// TestSearchCommand_MissingQueryShowsHelp verifies the intended behavior
|
||||
// TestSearchCommand_MissingQueryShowsHelp verifies the intended behavior.
|
||||
// NOT parallel: cmd.Help() calls cobra's OutOrStdout() which reads os.Stdout
|
||||
// even when cmd.SetOut() is set (it evaluates os.Stdout as the default argument).
|
||||
// This races with captureStdout() in parallel tests that redirect os.Stdout.
|
||||
func TestSearchCommand_MissingQueryShowsHelp(t *testing.T) {
|
||||
t.Parallel()
|
||||
// This test verifies that when query is missing, we:
|
||||
// 1. Print error message to stderr
|
||||
// 2. Show help (even if it fails, we tried)
|
||||
|
|
@ -154,6 +156,7 @@ func TestSearchCommand_MissingQueryShowsHelp(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
cmd.SetOut(&bytes.Buffer{}) // Prevent cmd.Help() from reading os.Stdout (races with captureStdout)
|
||||
cmd.SetErr(&errBuf)
|
||||
cmd.SetArgs([]string{}) // No query
|
||||
_ = cmd.Execute()
|
||||
|
|
|
|||
183
cmd/bd/stdio_race_guard_test.go
Normal file
183
cmd/bd/stdio_race_guard_test.go
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
//go:build cgo
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// cobraOutputMethods lists cobra.Command methods that read os.Stdout or
|
||||
// os.Stderr via OutOrStdout()/ErrOrStderr(). Cobra evaluates os.Stdout as
|
||||
// an argument even when cmd.SetOut() has been called, so calling ANY of
|
||||
// these in a parallel test races with captureStdout()/captureStderr().
|
||||
var cobraOutputMethods = []string{
|
||||
".Help(",
|
||||
".Execute(",
|
||||
".Print(",
|
||||
".Printf(",
|
||||
".Println(",
|
||||
".PrintErr(",
|
||||
".PrintErrf(",
|
||||
".PrintErrln(",
|
||||
".Usage(",
|
||||
".UsageString(",
|
||||
}
|
||||
|
||||
// TestCobraParallelPolicyGuard scans test source files and fails if any
|
||||
// parallel test calls cobra output methods. This prevents the data race
|
||||
// where cobra's OutOrStdout()/ErrOrStderr() reads os.Stdout/os.Stderr
|
||||
// concurrently with captureStdout()/captureStderr() redirecting them.
|
||||
//
|
||||
// The rule: if a test function contains t.Parallel() (in code, not
|
||||
// comments), it must NOT call any cobra output method. Setting
|
||||
// cmd.SetOut()/SetErr() does NOT prevent the race because cobra eagerly
|
||||
// evaluates os.Stdout as the default argument before checking outWriter.
|
||||
//
|
||||
// Fix options for flagged tests:
|
||||
// 1. Remove t.Parallel() (preferred for fast tests)
|
||||
// 2. Serialize the cobra call under stdioMutex
|
||||
//
|
||||
// This is intentionally blunt regex matching (80/20), not full AST analysis.
|
||||
func TestCobraParallelPolicyGuard(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
testFiles, err := filepath.Glob("*_test.go")
|
||||
if err != nil {
|
||||
t.Fatalf("glob: %v", err)
|
||||
}
|
||||
|
||||
funcPattern := regexp.MustCompile(`(?m)^func (Test\w+)\(`)
|
||||
|
||||
for _, file := range testFiles {
|
||||
data, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", file, err)
|
||||
}
|
||||
content := string(data)
|
||||
|
||||
matches := funcPattern.FindAllStringIndex(content, -1)
|
||||
names := funcPattern.FindAllStringSubmatch(content, -1)
|
||||
|
||||
for i, match := range matches {
|
||||
start := match[0]
|
||||
end := len(content)
|
||||
if i+1 < len(matches) {
|
||||
end = matches[i+1][0]
|
||||
}
|
||||
body := content[start:end]
|
||||
funcName := names[i][1]
|
||||
|
||||
// Strip comment lines to avoid false positives from
|
||||
// "// Not using t.Parallel() because ..." etc.
|
||||
stripped := stripLineComments(body)
|
||||
|
||||
if !strings.Contains(stripped, "t.Parallel()") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Also strip string literals containing method names (this
|
||||
// guard scans itself, so its string constants would match).
|
||||
strippedStrings := stripStringLiterals(stripped)
|
||||
|
||||
for _, method := range cobraOutputMethods {
|
||||
if strings.Contains(strippedStrings, method) {
|
||||
t.Errorf("%s:%s is t.Parallel() and calls cobra %s — "+
|
||||
"this races with captureStdout()/captureStderr() "+
|
||||
"because cobra eagerly reads os.Stdout/os.Stderr. "+
|
||||
"Remove t.Parallel() or serialize under stdioMutex. "+
|
||||
"See stdioMutex in test_helpers_test.go.",
|
||||
file, funcName, method)
|
||||
break // one error per function is enough
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stripLineComments removes // comment lines, preserving code lines.
|
||||
func stripLineComments(s string) string {
|
||||
var b strings.Builder
|
||||
for line := range strings.SplitSeq(s, "\n") {
|
||||
if strings.HasPrefix(strings.TrimSpace(line), "//") {
|
||||
continue
|
||||
}
|
||||
b.WriteString(line)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// stripStringLiterals replaces quoted strings with empty quotes to prevent
|
||||
// the guard from matching method names inside string constants.
|
||||
func stripStringLiterals(s string) string {
|
||||
var b strings.Builder
|
||||
inString := false
|
||||
escaped := false
|
||||
for _, ch := range s {
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if ch == '\\' && inString {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if ch == '"' {
|
||||
inString = !inString
|
||||
b.WriteRune(ch)
|
||||
continue
|
||||
}
|
||||
if !inString {
|
||||
b.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// TestStdioMutexContract verifies that stdioMutex actually serializes
|
||||
// captureStdout calls. If someone removes or bypasses the mutex, this
|
||||
// test fails deterministically without needing -race.
|
||||
func TestStdioMutexContract(t *testing.T) {
|
||||
var inside atomic.Int32
|
||||
var violations atomic.Int32
|
||||
|
||||
const goroutines = 4
|
||||
const iterations = 50
|
||||
|
||||
done := make(chan struct{}, goroutines)
|
||||
|
||||
for range goroutines {
|
||||
go func() {
|
||||
defer func() { done <- struct{}{} }()
|
||||
for range iterations {
|
||||
captureStdout(t, func() error {
|
||||
n := inside.Add(1)
|
||||
if n > 1 {
|
||||
violations.Add(1)
|
||||
}
|
||||
// Busyloop to widen the race window
|
||||
sum := 0
|
||||
for j := range 100 {
|
||||
sum += j
|
||||
}
|
||||
_ = sum
|
||||
inside.Add(-1)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
for range goroutines {
|
||||
<-done
|
||||
}
|
||||
|
||||
if v := violations.Load(); v > 0 {
|
||||
t.Fatalf("stdioMutex failed to serialize: %d concurrent entries detected", v)
|
||||
}
|
||||
}
|
||||
|
|
@ -33,6 +33,14 @@ var doltNewMutex sync.Mutex
|
|||
// stdioMutex serializes tests that redirect os.Stdout or os.Stderr.
|
||||
// These process-global file descriptors cannot be safely redirected from
|
||||
// concurrent goroutines.
|
||||
//
|
||||
// IMPORTANT: Any test that calls cobra's Help(), Execute(), or Print*()
|
||||
// MUST NOT be parallel (no t.Parallel()), OR must serialize those calls
|
||||
// under stdioMutex. Setting cmd.SetOut() is NOT sufficient because cobra's
|
||||
// OutOrStdout() eagerly evaluates os.Stdout as the default argument even
|
||||
// when outWriter is set — the Go race detector catches this read.
|
||||
//
|
||||
// TestCobraParallelPolicyGuard in stdio_race_guard_test.go enforces this.
|
||||
var stdioMutex sync.Mutex
|
||||
|
||||
// generateUniqueTestID creates a globally unique test ID using prefix, test name, and atomic counter.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue