fix(test): resolve cobra OutOrStdout data race in CI (#1916)

* fix(test): remove t.Parallel() from cobra Help tests to fix data race

cobra's OutOrStdout() eagerly evaluates os.Stdout as the default
argument even when cmd.SetOut() has been called, so any parallel test
calling cmd.Help() or cmd.Execute() races with captureStdout() which
redirects os.Stdout behind stdioMutex. Remove t.Parallel() from the
three TestSearchCommand_* tests that exercise this path.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* test: add cobra-parallel policy guard and stdio mutex contract test

- TestCobraParallelPolicyGuard: scans test files and fails if any
  parallel test calls cobra Help()/Execute() without SetOut/SetErr.
  Strips comments to avoid false positives.
- TestStdioMutexContract: deterministic reentrancy check proving
  stdioMutex serializes captureStdout calls (fails without -race).
- Update stdioMutex documentation to explain that SetOut is NOT
  sufficient to prevent the race (cobra eagerly reads os.Stdout).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* refactor(test): strengthen cobra-parallel policy guard

Remove the SetOut/SetErr escape hatch since cobra's OutOrStdout()
eagerly evaluates os.Stdout even when outWriter is set. The guard
now forbids ALL cobra output methods in parallel tests. Also adds
stripStringLiterals() to prevent self-matching false positives.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Joseph Turian 2026-02-20 12:22:25 -05:00 • committed by GitHub
commit ba29fdc1f7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 199 additions and 5 deletions

View file

@ -23,8 +23,8 @@ import (
// - Even if Help() fails (e.g., output redirection fails), we still exit with code 1
// - The error from Help() is rare (typically I/O errors writing to stderr)
// - Since we're already in an error state, ignoring Help() errors is acceptable
// NOT parallel: see TestSearchCommand_MissingQueryShowsHelp comment (cobra OutOrStdout race).
func TestSearchCommand_HelpErrorHandling(t *testing.T) {
t.Parallel()
// Create a test command similar to searchCmd
cmd := &cobra.Command{
Use: "search [query]",
@ -77,9 +77,9 @@ func TestSearchCommand_HelpErrorHandling(t *testing.T) {
})
}
// TestSearchCommand_HelpSuppression verifies that #nosec comment is appropriate
// TestSearchCommand_HelpSuppression verifies that #nosec comment is appropriate.
// NOT parallel: see TestSearchCommand_MissingQueryShowsHelp comment (cobra OutOrStdout race).
func TestSearchCommand_HelpSuppression(t *testing.T) {
t.Parallel()
// This test documents why ignoring cmd.Help() error is safe:
//
// 1. Help() is called in an error path (missing required argument)
@ -119,9 +119,11 @@ func (fw *failingWriter) Write(p []byte) (n int, err error) {
return 0, io.ErrClosedPipe // Simulate I/O error
}
// TestSearchCommand_MissingQueryShowsHelp verifies the intended behavior
// TestSearchCommand_MissingQueryShowsHelp verifies the intended behavior.
// NOT parallel: cmd.Help() calls cobra's OutOrStdout() which reads os.Stdout
// even when cmd.SetOut() is set (it evaluates os.Stdout as the default argument).
// This races with captureStdout() in parallel tests that redirect os.Stdout.
func TestSearchCommand_MissingQueryShowsHelp(t *testing.T) {
t.Parallel()
// This test verifies that when query is missing, we:
// 1. Print error message to stderr
// 2. Show help (even if it fails, we tried)
@ -154,6 +156,7 @@ func TestSearchCommand_MissingQueryShowsHelp(t *testing.T) {
},
}
cmd.SetOut(&bytes.Buffer{}) // Prevent cmd.Help() from reading os.Stdout (races with captureStdout)
cmd.SetErr(&errBuf)
cmd.SetArgs([]string{}) // No query
_ = cmd.Execute()

View file

@ -0,0 +1,183 @@
//go:build cgo
package main
import (
"os"
"path/filepath"
"regexp"
"strings"
"sync/atomic"
"testing"
)
// cobraOutputMethods lists cobra.Command methods that read os.Stdout or
// os.Stderr via OutOrStdout()/ErrOrStderr(). Cobra evaluates os.Stdout as
// an argument even when cmd.SetOut() has been called, so calling ANY of
// these in a parallel test races with captureStdout()/captureStderr().
var cobraOutputMethods = []string{
".Help(",
".Execute(",
".Print(",
".Printf(",
".Println(",
".PrintErr(",
".PrintErrf(",
".PrintErrln(",
".Usage(",
".UsageString(",
}
// TestCobraParallelPolicyGuard scans test source files and fails if any
// parallel test calls cobra output methods. This prevents the data race
// where cobra's OutOrStdout()/ErrOrStderr() reads os.Stdout/os.Stderr
// concurrently with captureStdout()/captureStderr() redirecting them.
//
// The rule: if a test function contains t.Parallel() (in code, not
// comments), it must NOT call any cobra output method. Setting
// cmd.SetOut()/SetErr() does NOT prevent the race because cobra eagerly
// evaluates os.Stdout as the default argument before checking outWriter.
//
// Fix options for flagged tests:
// 1. Remove t.Parallel() (preferred for fast tests)
// 2. Serialize the cobra call under stdioMutex
//
// This is intentionally blunt regex matching (80/20), not full AST analysis.
func TestCobraParallelPolicyGuard(t *testing.T) {
t.Parallel()
testFiles, err := filepath.Glob("*_test.go")
if err != nil {
t.Fatalf("glob: %v", err)
}
funcPattern := regexp.MustCompile(`(?m)^func (Test\w+)\(`)
for _, file := range testFiles {
data, err := os.ReadFile(file)
if err != nil {
t.Fatalf("read %s: %v", file, err)
}
content := string(data)
matches := funcPattern.FindAllStringIndex(content, -1)
names := funcPattern.FindAllStringSubmatch(content, -1)
for i, match := range matches {
start := match[0]
end := len(content)
if i+1 < len(matches) {
end = matches[i+1][0]
}
body := content[start:end]
funcName := names[i][1]
// Strip comment lines to avoid false positives from
// "// Not using t.Parallel() because ..." etc.
stripped := stripLineComments(body)
if !strings.Contains(stripped, "t.Parallel()") {
continue
}
// Also strip string literals containing method names (this
// guard scans itself, so its string constants would match).
strippedStrings := stripStringLiterals(stripped)
for _, method := range cobraOutputMethods {
if strings.Contains(strippedStrings, method) {
t.Errorf("%s:%s is t.Parallel() and calls cobra %s — "+
"this races with captureStdout()/captureStderr() "+
"because cobra eagerly reads os.Stdout/os.Stderr. "+
"Remove t.Parallel() or serialize under stdioMutex. "+
"See stdioMutex in test_helpers_test.go.",
file, funcName, method)
break // one error per function is enough
}
}
}
}
}
// stripLineComments removes // comment lines, preserving code lines.
func stripLineComments(s string) string {
var b strings.Builder
for line := range strings.SplitSeq(s, "\n") {
if strings.HasPrefix(strings.TrimSpace(line), "//") {
continue
}
b.WriteString(line)
b.WriteByte('\n')
}
return b.String()
}
// stripStringLiterals replaces quoted strings with empty quotes to prevent
// the guard from matching method names inside string constants.
func stripStringLiterals(s string) string {
var b strings.Builder
inString := false
escaped := false
for _, ch := range s {
if escaped {
escaped = false
continue
}
if ch == '\\' && inString {
escaped = true
continue
}
if ch == '"' {
inString = !inString
b.WriteRune(ch)
continue
}
if !inString {
b.WriteRune(ch)
}
}
return b.String()
}
// TestStdioMutexContract verifies that stdioMutex actually serializes
// captureStdout calls. If someone removes or bypasses the mutex, this
// test fails deterministically without needing -race.
func TestStdioMutexContract(t *testing.T) {
var inside atomic.Int32
var violations atomic.Int32
const goroutines = 4
const iterations = 50
done := make(chan struct{}, goroutines)
for range goroutines {
go func() {
defer func() { done <- struct{}{} }()
for range iterations {
captureStdout(t, func() error {
n := inside.Add(1)
if n > 1 {
violations.Add(1)
}
// Busyloop to widen the race window
sum := 0
for j := range 100 {
sum += j
}
_ = sum
inside.Add(-1)
return nil
})
}
}()
}
for range goroutines {
<-done
}
if v := violations.Load(); v > 0 {
t.Fatalf("stdioMutex failed to serialize: %d concurrent entries detected", v)
}
}

View file

@ -33,6 +33,14 @@ var doltNewMutex sync.Mutex
// stdioMutex serializes tests that redirect os.Stdout or os.Stderr.
// These process-global file descriptors cannot be safely redirected from
// concurrent goroutines.
//
// IMPORTANT: Any test that calls cobra's Help(), Execute(), or Print*()
// MUST NOT be parallel (no t.Parallel()), OR must serialize those calls
// under stdioMutex. Setting cmd.SetOut() is NOT sufficient because cobra's
// OutOrStdout() eagerly evaluates os.Stdout as the default argument even
// when outWriter is set — the Go race detector catches this read.
//
// TestCobraParallelPolicyGuard in stdio_race_guard_test.go enforces this.
var stdioMutex sync.Mutex
// generateUniqueTestID creates a globally unique test ID using prefix, test name, and atomic counter.