diff --git a/.bazelrc b/.bazelrc index df220ae3c..34ccf6cd6 100644 --- a/.bazelrc +++ b/.bazelrc @@ -33,6 +33,16 @@ test:prcore --test_tag_filters=-requires-docker,-embedded,-manual test:ci --config=prcore test:ci --keep_going --test_summary=terse +# Docker lane: only the requires-docker targets (pr.yml's container-backed +# jobs), on a host with a docker daemon. Race like those jobs, but no -short +# and no BEADS_TEST_SKIP=dolt, which would keep the containers from starting. +# Those targets are also tagged no-remote-exec, so they run on the host even +# with --config=remote-exec. +test:docker --@rules_go//go/config:race +test:docker --test_tag_filters=requires-docker +test:docker --build_tests_only +test:docker --keep_going + # Remote execution against any REAPI-compatible service. Only generic tuning # lives here; the executor endpoint and TLS credentials are never committed. # Developers put them in a gitignored user.bazelrc or .bazelrc.local; CI writes diff --git a/BUILD.bazel b/BUILD.bazel index e22749670..e97ad1716 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -56,11 +56,23 @@ filegroup( visibility = ["//test/docsync:__pkg__"], ) -# //scripts:scripts_test dolt pin tests (ci_workflow_test.go): the workflows -# must install the pinned dolt. +# //scripts:scripts_test workflow-policy and CI-script tests: the whole +# .github tree (workflows, composite actions, CI helper scripts) they parse +# and run. The one whole-tree input the plan allows (it is small). filegroup( name = "workflow_files", - srcs = glob([".github/workflows/*.yml"]), + srcs = glob([".github/**"]), + visibility = ["//scripts:__pkg__"], +) + +# //scripts:scripts_test runs scripts/test.sh, which sources the canonical +# build flags and reads the known-broken-test skip list. +filegroup( + name = "test_sh_files", + srcs = [ + ".buildflags", + ".test-skip", + ], visibility = ["//scripts:__pkg__"], ) diff --git a/backend/conformance/role_bundle_test.go b/backend/conformance/role_bundle_test.go index 7ef8ca9df..ce692298b 100644 --- a/backend/conformance/role_bundle_test.go +++ b/backend/conformance/role_bundle_test.go @@ -315,7 +315,7 @@ func TestRoleCasesFailsWhenAFactoryReturnsNil(t *testing.T) { func runProbeChild(t *testing.T, mode string) (string, error) { t.Helper() cmd := exec.Command(os.Args[0], "-test.run=^"+t.Name()+"$", "-test.v") - cmd.Env = append(os.Environ(), roleBundleProbeEnv+"="+mode) + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), roleBundleProbeEnv+"="+mode) out, err := cmd.CombinedOutput() return string(out), err } diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index f1e8faa6f..1267ccb2f 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -351,6 +351,7 @@ go_library( "//internal/atomicfile", "//internal/audit", "//internal/beads", + "//internal/ceiling", "//internal/compact", "//internal/config", "//internal/configfile", @@ -393,6 +394,7 @@ go_library( "//internal/storage/doltutil", "//internal/storage/domain", "//internal/storage/domain/db", + "//internal/storage/domain/git", "//internal/storage/embeddeddolt", "//internal/storage/externaldeps", "//internal/storage/fs", @@ -475,7 +477,9 @@ go_binary( go_test( name = "bd_test", - # -race -short runs ~10 min (pr-core gives go test -timeout=30m). + # -race -short runs ~4-10 min unsharded (pr-core gives go test + # -timeout=30m). Bazel shards by test count, not duration; test re-execs + # strip the shard variables (bazeltest.ShardFreeEnv). size = "large", timeout = "long", srcs = [ @@ -546,6 +550,7 @@ go_test( "config_get_backup_enabled_test.go", "config_guard_test.go", "config_isolation_regression_test.go", + "config_leaked_secret_test.go", "config_list_test.go", "config_nodb_embedded_test.go", "config_proxied_integration_test.go", @@ -766,6 +771,7 @@ go_test( "init_commit_test.go", "init_embedded_test.go", "init_gateway_test.go", + "init_git_bootstrap_routing_test.go", "init_guard_test.go", "init_hooks_test.go", "init_metrics_test.go", @@ -777,6 +783,8 @@ go_test( "init_proxied_server_test.go", "init_registered_backend_test.go", "init_remote_test.go", + "init_role_environment_test.go", + "init_role_routing_test.go", "init_safety_test.go", "init_server_flags_test.go", "init_stealth_test.go", @@ -1130,6 +1138,7 @@ go_test( "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", }, + shard_count = 16, deps = [ "//:beads", "//cmd/bd/doctor", @@ -1169,6 +1178,8 @@ go_test( "//internal/storage/domain", "//internal/storage/embeddeddolt", "//internal/storage/externaldeps", + "//internal/storage/fs", + "//internal/storage/git", "//internal/storage/issueops", "//internal/storage/journalscan", "//internal/storage/kvkeys", diff --git a/cmd/bd/close_last_touched_test.go b/cmd/bd/close_last_touched_test.go index 555d18328..501dce7d2 100644 --- a/cmd/bd/close_last_touched_test.go +++ b/cmd/bd/close_last_touched_test.go @@ -18,7 +18,8 @@ import ( // pollute parsed values (e.g. an issue ID from `bd q`). func runBDStdout(t *testing.T, binPath, workDir string, args ...string) string { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + // 2m, not 30s: a -race bd under a loaded CI host has taken >30s for one init. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() cmd := exec.CommandContext(ctx, binPath, args...) cmd.Dir = workDir diff --git a/cmd/bd/config.go b/cmd/bd/config.go index 95bcb450b..f39a33d77 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/viper" "github.com/steveyegge/beads/cmd/bd/doctor" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" @@ -1022,7 +1023,8 @@ func isValidRemoteURL(rawURL string) bool { // findBeadsRepoRoot walks up from the given path to find the repo root (containing .beads) func findBeadsRepoRoot(startPath string) string { path := startPath - for { + bound := ceiling.For(startPath) + for !bound.Excludes(path) { beadsDir := filepath.Join(path, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { return path diff --git a/cmd/bd/context_identity_test.go b/cmd/bd/context_identity_test.go index c8d9d405f..6e166d370 100644 --- a/cmd/bd/context_identity_test.go +++ b/cmd/bd/context_identity_test.go @@ -11,6 +11,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/backends" "github.com/steveyegge/beads/internal/storage/contextinfo" @@ -83,8 +84,13 @@ func TestContextRoutesNameOneWorkspaceTheSameWay(t *testing.T) { } t.Chdir(dir) t.Setenv("BEADS_DIR", beadsDir) + // git caches the cwd's repository context process-wide; a test + // that ran earlier from a non-repo directory would otherwise make + // this fresh repo look like "not a git repository". + git.ResetCaches() beads.ResetCaches() t.Cleanup(beads.ResetCaches) + t.Cleanup(git.ResetCaches) provider, err := contextinfo.NewContextProvider(dir, Version).ContextUseCase().GetContextInfo(t.Context()) if err != nil { diff --git a/cmd/bd/create_repo_remote_url_preprun_test.go b/cmd/bd/create_repo_remote_url_preprun_test.go index 1233e3177..e03bc0b2e 100644 --- a/cmd/bd/create_repo_remote_url_preprun_test.go +++ b/cmd/bd/create_repo_remote_url_preprun_test.go @@ -6,6 +6,9 @@ import ( "path/filepath" "strings" "testing" + + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/git" ) // TestCreateRemoteRepoSkipsLocalDatabaseGuard is a regression test for a gap @@ -54,6 +57,18 @@ func TestCreateRemoteRepoSkipsLocalDatabaseGuard(t *testing.T) { t.Fatalf("chdir(%q): %v", noBeadsCwd, err) } t.Cleanup(func() { _ = os.Chdir(originalWD) }) + // git and beads cache the cwd's repository and workspace process-wide. Start + // from caches that describe noBeadsCwd: a cache an earlier test filled from + // the package directory (a checkout, or a worktree whose main repository has + // a .beads) would hand create that workspace instead. Reset again on the + // way out, before the cwd is restored, so the next test does not inherit + // this directory's answers. + git.ResetCaches() + beads.ResetCaches() + t.Cleanup(func() { + git.ResetCaches() + beads.ResetCaches() + }) // Hide the `dolt` CLI so remotecache.Ensure fails fast and // deterministically in CGO-enabled builds, without attempting a real network diff --git a/cmd/bd/doctor/BUILD.bazel b/cmd/bd/doctor/BUILD.bazel index c3d3a4194..73f72e14e 100644 --- a/cmd/bd/doctor/BUILD.bazel +++ b/cmd/bd/doctor/BUILD.bazel @@ -62,6 +62,7 @@ go_library( "//internal/doltremote", "//internal/doltserver", "//internal/git", + "//internal/gitenv", "//internal/gitignore", "//internal/storage", "//internal/storage/dberrors", @@ -154,6 +155,7 @@ go_test( "//internal/configfile", "//internal/doltserver", "//internal/git", + "//internal/gitenv", "//internal/storage/dolt", "//internal/storage/doltutil", "//internal/storage/embeddeddolt", diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel index 63a6738f6..4ca96cb1d 100644 --- a/cmd/bd/doctor/fix/BUILD.bazel +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "fix", @@ -96,6 +97,30 @@ go_test( ], ) +# The Dolt-backed half of fix_test, as pr.yml's "Test doctor/fix (Dolt-backed, +# hard-require container)" job runs it: TestMain starts a dolt sql-server +# container through testcontainers, and BEADS_FIX_REQUIRE_DOLT=1 turns a +# missing container into a failure instead of a silent skip. fix_test itself +# stays untagged: under --config=prcore (BEADS_TEST_SKIP=dolt, like pr-core's +# `go test`) it never touches docker and runs anywhere. +# +# Tags: +# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, +# selected by --config=docker. +# no-remote-exec: the daemon is on the host, not on remote workers. +sh_test( + name = "fix_docker_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :fix_test)"], + data = [":fix_test"], + env = {"BEADS_FIX_REQUIRE_DOLT": "1"}, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/cmd/bd/doctor/fix/common.go b/cmd/bd/doctor/fix/common.go index 5b6eb96ea..a5f751417 100644 --- a/cmd/bd/doctor/fix/common.go +++ b/cmd/bd/doctor/fix/common.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" "strings" + "testing" "github.com/steveyegge/beads/internal/beads" ) @@ -38,9 +39,8 @@ func getBdBinary() (string, error) { } // Check if we're running as a test binary - this prevents fork bombs - // when tests call functions that execute bd subcommands - baseName := filepath.Base(exe) - if strings.HasSuffix(baseName, ".test") || strings.Contains(baseName, ".test.") { + // when tests call functions that execute bd subcommands. + if isTestBinary(exe) { return "", ErrTestBinary } @@ -56,6 +56,18 @@ func getBdBinary() (string, error) { return bdPath, nil } +// isTestBinary reports whether exe is a Go test binary. testing.Testing() is +// the authoritative signal: both `go test` and rules_go link test binaries with +// it set, whatever the file is called (Bazel names them _test, not +// .test). The name check is kept for binaries built by other means. +func isTestBinary(exe string) bool { + if testing.Testing() { + return true + } + baseName := filepath.Base(exe) + return strings.HasSuffix(baseName, ".test") || strings.Contains(baseName, ".test.") +} + // validateBeadsWorkspace ensures the path is a valid beads workspace before // attempting any fix operations. This prevents path traversal attacks. func validateBeadsWorkspace(path string) error { diff --git a/cmd/bd/doctor/fix/e2e_test.go b/cmd/bd/doctor/fix/e2e_test.go index 50bd17a75..5183c01c6 100644 --- a/cmd/bd/doctor/fix/e2e_test.go +++ b/cmd/bd/doctor/fix/e2e_test.go @@ -103,6 +103,21 @@ func TestGetBdBinary_Errors(t *testing.T) { }) } +// TestGetBdBinary_DetectsTestBinaryByAnyName guards against a fork bomb: a test +// binary must never be returned as "bd", whatever it is named. Bazel names +// test binaries _test (no ".test" suffix); before the guard used +// testing.Testing(), each fix test re-ran the whole suite recursively there. +func TestGetBdBinary_DetectsTestBinaryByAnyName(t *testing.T) { + if _, err := getBdBinary(); !errors.Is(err, ErrTestBinary) { + t.Fatalf("getBdBinary() in a test binary: got err %v, want ErrTestBinary", err) + } + for _, name := range []string{"/x/fix.test", "/x/fix_test", "/x/bd"} { + if !isTestBinary(name) { + t.Errorf("isTestBinary(%q) = false inside a test binary", name) + } + } +} + // TestFilePermissionErrors tests handling of file permission issues func TestFilePermissionErrors(t *testing.T) { if os.Getuid() == 0 { diff --git a/cmd/bd/legacy_upgrade_guard.go b/cmd/bd/legacy_upgrade_guard.go index 02c8e36e0..063146624 100644 --- a/cmd/bd/legacy_upgrade_guard.go +++ b/cmd/bd/legacy_upgrade_guard.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/git" @@ -245,7 +246,11 @@ func guardUndiscoveredLegacyWorkspace() error { } dir := utils.CanonicalizePath(cwd) boundary := utils.CanonicalizePath(git.GetRepoRoot()) + bound := ceiling.For(dir) for { + if bound.Excludes(dir) { + return nil + } if err := guardLegacyUpgradeWorkspace(filepath.Join(dir, ".beads")); err != nil { return err } diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 260a14774..7880c931c 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -24,6 +24,7 @@ import ( "github.com/subosito/gotenv" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/debug" @@ -789,7 +790,8 @@ func resolveCommandBeadsDir(dbPath string) string { return beadsDir } - for dir := filepath.Dir(dbPath); dir != "" && dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + bound := ceiling.For(filepath.Dir(dbPath)) + for dir := filepath.Dir(dbPath); dir != "" && dir != filepath.Dir(dir) && !bound.Excludes(dir); dir = filepath.Dir(dir) { candidate := filepath.Join(dir, ".beads") if info, err := os.Stat(candidate); err == nil && info.IsDir() { return candidate diff --git a/cmd/bd/prime_gemini_hook_test.go b/cmd/bd/prime_gemini_hook_test.go index fb74380c9..1c8816d96 100644 --- a/cmd/bd/prime_gemini_hook_test.go +++ b/cmd/bd/prime_gemini_hook_test.go @@ -50,7 +50,8 @@ func runPrimeBinary(t *testing.T, binPath, workDir string, args ...string) (stdo // .beads/ directory so FindBeadsDir succeeds. func initBeadsWorkspace(t *testing.T, binPath, workDir string) { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + // 2m, not 30s: a -race bd under a loaded CI host has taken >30s for one init. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() cmd := exec.CommandContext(ctx, binPath, "init", "--prefix", "test") cmd.Dir = workDir diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index f6a03b25f..af19356a0 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "protocol", @@ -49,3 +50,34 @@ go_test( "//internal/testutil/bazeltest", ], ) + +# protocol_test as pr.yml's "Contract corpus" job runs it: TestMain starts a +# dolt sql-server container and BEADS_PROTOCOL_REQUIRE_DOLT=1 turns a missing +# container into a failure instead of a skip. It repeats protocol_test's `env` +# (go_test_variant.sh cannot inherit it). protocol_test itself stays untagged: +# under --config=prcore (BEADS_TEST_SKIP=dolt) its Dolt-backed tests skip. +# +# Tags: +# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, +# selected by --config=docker. +# no-remote-exec: the daemon is on the host, not on remote workers. +sh_test( + name = "protocol_docker_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :protocol_test)"], + data = [ + ":protocol_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_PROTOCOL_REQUIRE_DOLT": "1", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + }, + # ~10 min unsharded without -race in pr.yml. + shard_count = 8, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) diff --git a/cmd/bd/store_reopen.go b/cmd/bd/store_reopen.go index a72bbd96b..f65f69255 100644 --- a/cmd/bd/store_reopen.go +++ b/cmd/bd/store_reopen.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/utils" @@ -63,7 +64,8 @@ func resolveBeadsDirForDBPath(dbPath string) string { } addAncestorCandidates := func(path string) { - for dir := path; dir != "" && dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + bound := ceiling.For(path) + for dir := path; dir != "" && dir != filepath.Dir(dir) && !bound.Excludes(dir); dir = filepath.Dir(dir) { addCandidate(filepath.Join(dir, ".beads")) if filepath.Base(dir) == ".beads" { addCandidate(dir) diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index e129d0b55..0359f124d 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -50,7 +50,9 @@ type testRunner interface { } func runTestsAndSweep(m testRunner) int { + stdout, stderr := os.Stdout, os.Stderr code := m.Run() + code = checkStdioAfterRun(code, stdout, stderr) swept := doltserver.SweepSuiteTestServers(testTempRoot) return doltserver.ApplyLeakPolicy("cmd/bd", code, swept) } diff --git a/cmd/bd/worktree_remove_test.go b/cmd/bd/worktree_remove_test.go index 8a3b90dcc..b30b3c8b3 100644 --- a/cmd/bd/worktree_remove_test.go +++ b/cmd/bd/worktree_remove_test.go @@ -11,6 +11,8 @@ import ( "reflect" "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) const ( @@ -295,7 +297,7 @@ func runWorktreeRemoveProcess(t *testing.T, dir string, extraEnv []string, args } command := exec.Command(executable, "-test.run=^TestWorktreeRemoveProcessHelper$") command.Dir = dir - command.Env = overrideWorktreeRemoveEnv(os.Environ(), append([]string{worktreeRemoveHelperEnv + "=1", worktreeRemoveHelperArgsEnv + "=" + string(encodedArgs), "BD_DISABLE_METRICS=1"}, extraEnv...)) + command.Env = overrideWorktreeRemoveEnv(bazeltest.ShardFreeEnv(os.Environ()), append([]string{worktreeRemoveHelperEnv + "=1", worktreeRemoveHelperArgsEnv + "=" + string(encodedArgs), "BD_DISABLE_METRICS=1"}, extraEnv...)) var stdout, stderr bytes.Buffer command.Stdout, command.Stderr = &stdout, &stderr err = command.Run() diff --git a/cmd/bd/zz_stdio_leak_guard_test.go b/cmd/bd/zz_stdio_leak_guard_test.go index b5da2c2bc..66bfc4cbd 100644 --- a/cmd/bd/zz_stdio_leak_guard_test.go +++ b/cmd/bd/zz_stdio_leak_guard_test.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "os" "testing" ) @@ -14,9 +15,16 @@ import ( // the var-init streams: under `go test -json` the testing framework swaps // os.Stderr to os.Stdout inside M.Run (go.dev/issue/33419), after var-init // and before any test, and that framework swap is not a leak (#5881). +// +// It only sees tests that ran before it in this process. TestMain repeats the +// check after m.Run (checkStdioAfterRun), which covers every test in the +// process, including each Bazel shard, where this test runs in one shard only. func TestZZStdioNotLeaked(t *testing.T) { if baselineStdout == nil || baselineStderr == nil { - t.Skip("stdio baseline not captured (TestAAAStdioBaseline filtered out); nothing to compare against") + // TestAAAStdioBaseline did not run in this process (-run filtered it + // out, or Bazel sharding put it in another shard); TestMain's check + // after m.Run still applies. + t.Skip("no baseline in this process; checkStdioAfterRun covers it") } if os.Stdout != baselineStdout { t.Errorf("os.Stdout was leaked by an earlier test (now fd=%d name=%q); "+ @@ -31,3 +39,29 @@ func TestZZStdioNotLeaked(t *testing.T) { os.Stderr = baselineStderr } } + +// checkStdioAfterRun fails the process (returning a non-zero code) when a test +// left os.Stdout or os.Stderr reassigned. stdout/stderr are the streams from +// before m.Run. The one reassignment allowed is the framework's own under +// -json, os.Stderr = os.Stdout (see TestZZStdioNotLeaked). +func checkStdioAfterRun(code int, stdout, stderr *os.File) int { + var leaked []string + if os.Stdout != stdout { + leaked = append(leaked, fmt.Sprintf("os.Stdout (now fd=%d name=%q)", os.Stdout.Fd(), os.Stdout.Name())) + } + if os.Stderr != stderr && os.Stderr != stdout { + leaked = append(leaked, fmt.Sprintf("os.Stderr (now fd=%d name=%q)", os.Stderr.Fd(), os.Stderr.Name())) + } + if len(leaked) == 0 { + return code + } + os.Stdout, os.Stderr = stdout, stderr + for _, l := range leaked { + fmt.Fprintf(stderr, "FAIL: %s was leaked by a test in this process; "+ + "a capture helper restored it on the happy path only - move the restore into a defer\n", l) + } + if code == 0 { + code = 1 + } + return code +} diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 959618da9..88915e2fb 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -499,6 +499,7 @@ Selected commonly-used variables: | `BD_DEBUG` | Enable debug logging | | `BD_MIGRATION_FREEZE_FILE` | Check this exact path for the freeze marker instead of walking ancestor directories; authoritative when set (see [Migration Freeze](#migration-freeze)) | | `BEADS_DIR` | Force the active beads workspace directory | +| `BEADS_CEILING_DIRECTORIES` | Directories (separated like `PATH`) that `.beads` and `config.yaml` discovery never looks at or above, like git's `GIT_CEILING_DIRECTORIES`; the starting directory is always examined. For sandboxes such as `bazel test` that must not reach the user's own `~/.beads` | | `BEADS_ACTOR` | Actor identity (preferred over `BD_ACTOR`, which is a deprecated alias) | | `BEADS_IDENTITY` | Sender identity for `bd mail` | | `BEADS_FSCK_TIMEOUT` | Runtime-only timeout for the pre-push `dolt fsck --quiet` integrity check (default `30s`) | diff --git a/internal/beads/BUILD.bazel b/internal/beads/BUILD.bazel index 94bdfadff..9c27bb763 100644 --- a/internal/beads/BUILD.bazel +++ b/internal/beads/BUILD.bazel @@ -10,8 +10,10 @@ go_library( importpath = "github.com/steveyegge/beads/internal/beads", visibility = ["//:__subpackages__"], deps = [ + "//internal/ceiling", "//internal/configfile", "//internal/git", + "//internal/gitenv", "//internal/storage", "//internal/storage/backends", "//internal/utils", @@ -34,9 +36,11 @@ go_test( ], embed = [":beads"], deps = [ + "//internal/ceiling", "//internal/configfile", "//internal/doltserver", "//internal/git", + "//internal/gitenv", "//internal/testutil", "//internal/utils", ], diff --git a/internal/beads/beads.go b/internal/beads/beads.go index bf153a423..7113d75dd 100644 --- a/internal/beads/beads.go +++ b/internal/beads/beads.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" @@ -786,10 +787,14 @@ func hasBeadsProjectFiles(beadsDir string) bool { // when they agree on which ancestors exist. The ceiling is not extended to "/" // because it guards against ambient state in a world-writable shared root, // which the filesystem root is not. +// +// BEADS_CEILING_DIRECTORIES (package ceiling) also ends the walk: no directory +// at or above a ceiling containing the origin is yielded. type AncestorDirWalk struct { next string origin string tempRoot string + bound *ceiling.Bound done bool } @@ -797,10 +802,12 @@ type AncestorDirWalk struct { // caller's actual discovery start even when startDir begins a later segment of // a bounded walk. func NewAncestorDirWalk(startDir, originDir string) *AncestorDirWalk { + origin := canonicalizeAncestorWalkPath(originDir) return &AncestorDirWalk{ next: canonicalizeAncestorWalkPath(startDir), - origin: canonicalizeAncestorWalkPath(originDir), + origin: origin, tempRoot: canonicalizeAncestorWalkPath(os.TempDir()), + bound: ceiling.For(origin), } } @@ -832,12 +839,17 @@ func canonicalizeAncestorWalkPath(path string) string { } } -// Next returns the next directory permitted by the temp-root ceiling. +// Next returns the next directory permitted by the temp-root and +// BEADS_CEILING_DIRECTORIES ceilings. func (w *AncestorDirWalk) Next() (string, bool) { if w == nil || w.done || w.next == "" || w.next == "." { return "", false } dir := w.next + if w.bound.Excludes(dir) { + w.done = true + return "", false + } if w.tempRoot != "" && dir == w.tempRoot && dir != w.origin { w.done = true return "", false diff --git a/internal/beads/beads_test.go b/internal/beads/beads_test.go index b4e64378d..555ee0fc9 100644 --- a/internal/beads/beads_test.go +++ b/internal/beads/beads_test.go @@ -12,6 +12,7 @@ import ( "strings" "testing" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" @@ -593,6 +594,33 @@ func TestAncestorDirWalkEndsAtOSTempRoot(t *testing.T) { } } +// TestAncestorDirWalkStopsBelowBeadsCeiling pins that BEADS_CEILING_DIRECTORIES +// ends every FindBeadsDir-style walk below the ceiling, so a .beads at or +// above it is never found. +func TestAncestorDirWalkStopsBelowBeadsCeiling(t *testing.T) { + sandbox := t.TempDir() + ceilingDir := filepath.Join(sandbox, "ceiling") + start := filepath.Join(ceilingDir, "project", "nested") + if err := os.MkdirAll(start, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", filepath.Join(sandbox, "elsewhere")) + t.Setenv(ceiling.EnvVar, ceilingDir) + + var got []string + walk := NewAncestorDirWalk(start, start) + for dir, ok := walk.Next(); ok; dir, ok = walk.Next() { + got = append(got, dir) + } + want := []string{ + canonicalizeAncestorWalkPath(start), + canonicalizeAncestorWalkPath(filepath.Join(ceilingDir, "project")), + } + if !slices.Equal(got, want) { + t.Fatalf("walk yielded %q, want %q", got, want) + } +} + // TestAncestorDirWalkYieldsFilesystemRoot pins the documented choice that the // walk includes the filesystem root. Most of the hand-rolled loops this type // replaced stopped before "/"; the unified walk does not, so the behavior is @@ -603,8 +631,10 @@ func TestAncestorDirWalkYieldsFilesystemRoot(t *testing.T) { if err := os.MkdirAll(start, 0o755); err != nil { t.Fatal(err) } - // Point the ceiling somewhere off this walk so it can reach the root. + // Point the ceiling somewhere off this walk so it can reach the root, and + // lift any BEADS_CEILING_DIRECTORIES the test runner set. t.Setenv("TMPDIR", filepath.Join(sandbox, "tmp")) + t.Setenv(ceiling.EnvVar, "") fsRoot := canonicalizeAncestorWalkPath(start) for { diff --git a/internal/ceiling/BUILD.bazel b/internal/ceiling/BUILD.bazel new file mode 100644 index 000000000..9ef453ece --- /dev/null +++ b/internal/ceiling/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "ceiling", + srcs = ["ceiling.go"], + importpath = "github.com/steveyegge/beads/internal/ceiling", + visibility = ["//:__subpackages__"], +) + +go_test( + name = "ceiling_test", + srcs = ["ceiling_test.go"], + embed = [":ceiling"], +) diff --git a/internal/ceiling/ceiling.go b/internal/ceiling/ceiling.go new file mode 100644 index 000000000..ef8e00c98 --- /dev/null +++ b/internal/ceiling/ceiling.go @@ -0,0 +1,131 @@ +// Package ceiling bounds the upward directory searches bd uses to discover a +// .beads workspace or its config.yaml. +// +// BEADS_CEILING_DIRECTORIES is a list of absolute directories separated by the +// OS path-list separator (":" on POSIX, ";" on Windows), like git's +// GIT_CEILING_DIRECTORIES. A search that starts inside a ceiling never examines +// the ceiling itself or anything above it; the start directory is always +// examined. Ceilings that do not contain the start have no effect, and empty or +// relative entries are ignored. When the variable is unset nothing changes. +// +// It exists for sandboxes such as `bazel test`, whose working directory lives +// below the developer's home: without a ceiling, discovery walks out of the +// sandbox and reads or writes the developer's own ~/.beads. +package ceiling + +import ( + "os" + "path/filepath" + "strings" +) + +// EnvVar names the ceiling list. +const EnvVar = "BEADS_CEILING_DIRECTORIES" + +// Bound is the ceiling of one upward search. A nil Bound permits every +// directory, so callers can use it unconditionally. +type Bound struct { + start []string + ceilings []string +} + +// For returns the bound for a search starting at start, or nil when +// BEADS_CEILING_DIRECTORIES names no ceiling containing start. +func For(start string) *Bound { + list := os.Getenv(EnvVar) + if list == "" { + return nil + } + startForms := forms(start) + var ceilings []string + for _, entry := range filepath.SplitList(list) { + if entry == "" || !filepath.IsAbs(entry) { + continue + } + for _, c := range forms(entry) { + if anyWithin(startForms, c) { + ceilings = append(ceilings, c) + } + } + } + if len(ceilings) == 0 { + return nil + } + return &Bound{start: startForms, ceilings: ceilings} +} + +// Excludes reports whether the search must stop before examining dir: dir is +// not the start and is a ceiling or lies above one. +func (b *Bound) Excludes(dir string) bool { + if b == nil { + return false + } + dirForms := forms(dir) + for _, d := range dirForms { + for _, s := range b.start { + if d == s { + return false + } + } + } + for _, c := range b.ceilings { + for _, d := range dirForms { + if within(c, d) { + return true + } + } + } + return false +} + +// GitEnv returns env with GIT_CEILING_DIRECTORIES set to the beads ceilings, +// for git probes that locate a workspace (and have scrubbed inherited git +// routing). env is returned unchanged when BEADS_CEILING_DIRECTORIES is unset. +func GitEnv(env []string) []string { + list := os.Getenv(EnvVar) + if list == "" { + return env + } + out := make([]string, 0, len(env)+1) + for _, e := range env { + if !strings.HasPrefix(e, "GIT_CEILING_DIRECTORIES=") { + out = append(out, e) + } + } + return append(out, "GIT_CEILING_DIRECTORIES="+list) +} + +// forms returns the cleaned absolute path and, when it differs, its +// symlink-resolved form, so a ceiling and a walk that spell the same directory +// differently still meet. +func forms(path string) []string { + abs, err := filepath.Abs(path) + if err != nil { + return []string{filepath.Clean(path)} + } + abs = filepath.Clean(abs) + if resolved, err := filepath.EvalSymlinks(abs); err == nil && resolved != abs { + return []string{abs, resolved} + } + return []string{abs} +} + +// anyWithin reports whether any of paths is parent or lies below it. +func anyWithin(paths []string, parent string) bool { + for _, p := range paths { + if within(p, parent) { + return true + } + } + return false +} + +func within(path, parent string) bool { + if path == parent { + return true + } + if !strings.HasSuffix(parent, string(filepath.Separator)) { + parent += string(filepath.Separator) + } + return strings.HasPrefix(path, parent) +} diff --git a/internal/ceiling/ceiling_test.go b/internal/ceiling/ceiling_test.go new file mode 100644 index 000000000..d5560f928 --- /dev/null +++ b/internal/ceiling/ceiling_test.go @@ -0,0 +1,102 @@ +package ceiling + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +// walk mimics a discovery loop: every directory from start upward that the +// bound lets it examine. +func walk(start string) []string { + b := For(start) + var seen []string + for dir := start; ; dir = filepath.Dir(dir) { + if b.Excludes(dir) { + break + } + seen = append(seen, dir) + if filepath.Dir(dir) == dir { + break + } + } + return seen +} + +func TestUnsetExaminesEveryAncestor(t *testing.T) { + t.Setenv(EnvVar, "") + root := t.TempDir() + start := filepath.Join(root, "a", "b") + if For(start) != nil { + t.Fatal("For must return nil when the variable is unset") + } + got := walk(start) + if !slices.Contains(got, root) || got[len(got)-1] != filepath.Dir(got[len(got)-1]) { + t.Fatalf("unset ceiling must reach the filesystem root, got %v", got) + } +} + +func TestCeilingStopsBelowItself(t *testing.T) { + root := t.TempDir() + start := filepath.Join(root, "a", "b") + if err := os.MkdirAll(start, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv(EnvVar, strings.Join([]string{"relative/ignored", "", root}, string(os.PathListSeparator))) + got := walk(start) + want := []string{start, filepath.Join(root, "a")} + if !slices.Equal(got, want) { + t.Fatalf("walk = %v, want %v", got, want) + } +} + +func TestStartAtCeilingIsExaminedButNothingAbove(t *testing.T) { + root := t.TempDir() + t.Setenv(EnvVar, root) + if got := walk(root); !slices.Equal(got, []string{root}) { + t.Fatalf("walk = %v, want only the start", got) + } +} + +func TestUnrelatedCeilingHasNoEffect(t *testing.T) { + root := t.TempDir() + other := filepath.Join(root, "other") + start := filepath.Join(root, "a") + t.Setenv(EnvVar, other) + if For(start) != nil { + t.Fatal("a ceiling that does not contain the start must not bound it") + } +} + +func TestCeilingMatchesThroughSymlink(t *testing.T) { + root := t.TempDir() + realDir := filepath.Join(root, "realDir") + start := filepath.Join(realDir, "a") + if err := os.MkdirAll(start, 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "link") + if err := os.Symlink(realDir, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + t.Setenv(EnvVar, link) + got := walk(start) + if !slices.Equal(got, []string{start}) { + t.Fatalf("walk = %v, want only %s (ceiling named through a symlink)", got, start) + } +} + +func TestGitEnv(t *testing.T) { + env := []string{"A=1", "GIT_CEILING_DIRECTORIES=/old"} + t.Setenv(EnvVar, "") + if got := GitEnv(env); !slices.Equal(got, env) { + t.Fatalf("unset: GitEnv = %v, want unchanged", got) + } + t.Setenv(EnvVar, "/x") + want := []string{"A=1", "GIT_CEILING_DIRECTORIES=/x"} + if got := GitEnv(env); !slices.Equal(got, want) { + t.Fatalf("GitEnv = %v, want %v", got, want) + } +} diff --git a/internal/config/BUILD.bazel b/internal/config/BUILD.bazel index 1474e1f71..b0d181fd1 100644 --- a/internal/config/BUILD.bazel +++ b/internal/config/BUILD.bazel @@ -17,6 +17,7 @@ go_library( importpath = "github.com/steveyegge/beads/internal/config", visibility = ["//:__subpackages__"], deps = [ + "//internal/ceiling", "//internal/debug", "//internal/gitenv", "@com_github_spf13_viper//:viper", diff --git a/internal/config/config.go b/internal/config/config.go index f6d5379f6..7a209329b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,6 +9,7 @@ import ( "time" "github.com/spf13/viper" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/gitenv" "gopkg.in/yaml.v3" @@ -139,10 +140,12 @@ func Initialize() error { cwd, err := os.Getwd() if err == nil { + // BEADS_CEILING_DIRECTORIES bounds both upward walks below. + bound := ceiling.For(cwd) var moduleRoot string if ignoreRepoConfig { // Find module root by walking up to go.mod. - for dir := cwd; dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + for dir := cwd; dir != filepath.Dir(dir) && !bound.Excludes(dir); dir = filepath.Dir(dir) { if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { moduleRoot = dir break @@ -172,7 +175,7 @@ func Initialize() error { } // Walk up parent directories to find .beads/config.yaml. - for dir := cwd; dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + for dir := cwd; dir != filepath.Dir(dir) && !bound.Excludes(dir); dir = filepath.Dir(dir) { p := filepath.Join(dir, ".beads", "config.yaml") if _, err := os.Stat(p); err == nil { // When BEADS_DIR points at a different runtime workspace, do not @@ -442,8 +445,9 @@ func gitDirsForRepo(repoPath string) (gitDir, commonDir string, ok bool) { // stops, so dropping discovery ceilings can select a containing parent // repository above repoPath. That is the intended trade, and it applies here // too -- repoPath is the process working directory, not a proven repository - // root. - cmd.Env = gitenv.ScrubRouting(os.Environ()) + // root. An explicit BEADS_CEILING_DIRECTORIES is the exception: it is + // reapplied as the git ceiling (a no-op when unset). + cmd.Env = ceiling.GitEnv(gitenv.ScrubRouting(os.Environ())) output, err := cmd.Output() if err != nil { return "", "", false diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2451584e7..9245e741b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -282,6 +282,46 @@ func TestInitialize_IgnoresModuleRootConfigWhenRequested(t *testing.T) { } } +// TestInitialize_StopsAtBeadsCeiling pins that BEADS_CEILING_DIRECTORIES keeps +// the config.yaml walk from reaching a .beads at or above the ceiling — here +// an unparseable one that would fail Initialize if it were read. +func TestInitialize_StopsAtBeadsCeiling(t *testing.T) { + restore := envSnapshot(t) + defer restore() + + tmpDir := t.TempDir() + homeDir := filepath.Join(tmpDir, "home") + configDir := filepath.Join(tmpDir, "xdg-config") + ceilingDir := filepath.Join(tmpDir, "ceiling") + workDir := filepath.Join(ceilingDir, "work") + for _, dir := range []string{homeDir, configDir, workDir, filepath.Join(tmpDir, ".beads"), filepath.Join(ceilingDir, ".beads")} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } + for _, dir := range []string{tmpDir, ceilingDir} { + if err := os.WriteFile(filepath.Join(dir, ".beads", "config.yaml"), []byte("json: [unclosed\n"), 0o644); err != nil { + t.Fatal(err) + } + } + + t.Setenv("HOME", homeDir) + t.Setenv("XDG_CONFIG_HOME", configDir) + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "") + t.Setenv("BEADS_CEILING_DIRECTORIES", ceilingDir) + t.Chdir(workDir) + + ResetForTesting() + defer ResetForTesting() + + if err := Initialize(); err != nil { + t.Fatalf("Initialize() read config at or above the ceiling: %v", err) + } + if got := ConfigFileUsed(); got != "" { + t.Fatalf("ConfigFileUsed() = %q, want none below the ceiling", got) + } +} + func TestLocalConfigOverride(t *testing.T) { // Isolate from environment variables restore := envSnapshot(t) diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index 9429a92ba..b19969463 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/gitenv" "gopkg.in/yaml.v3" ) @@ -750,7 +751,8 @@ func findProjectBeadsDir() string { return "" } - for dir := cwd; dir != filepath.Dir(dir); dir = filepath.Dir(dir) { + bound := ceiling.For(cwd) + for dir := cwd; dir != filepath.Dir(dir) && !bound.Excludes(dir); dir = filepath.Dir(dir) { beadsDir := filepath.Join(dir, ".beads") if info, err := os.Stat(beadsDir); err == nil && info.IsDir() { return beadsDir diff --git a/internal/linear/BUILD.bazel b/internal/linear/BUILD.bazel index e0c952a91..b5a2fcf3d 100644 --- a/internal/linear/BUILD.bazel +++ b/internal/linear/BUILD.bazel @@ -57,6 +57,7 @@ go_test( ] + select({ "@rules_go//go/platform:windows": [ "//internal/lockfile", + "//internal/testutil/bazeltest", "@org_golang_x_sys//windows", ], "//conditions:default": [], diff --git a/internal/linear/synclock_info_windows_test.go b/internal/linear/synclock_info_windows_test.go index ff2e0ae67..07ffed2f0 100644 --- a/internal/linear/synclock_info_windows_test.go +++ b/internal/linear/synclock_info_windows_test.go @@ -19,6 +19,7 @@ import ( "time" "github.com/steveyegge/beads/internal/lockfile" + "github.com/steveyegge/beads/internal/testutil/bazeltest" "golang.org/x/sys/windows" ) @@ -756,7 +757,7 @@ type syncLockHelperProcess struct { func startSyncLockHelper(t *testing.T, dir, mode string) *syncLockHelperProcess { t.Helper() cmd := exec.Command(os.Args[0], "-test.run=^TestSyncLockWindowsSubprocessHelper$") - cmd.Env = append(os.Environ(), + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), syncLockHelperModeEnv+"="+mode, syncLockHelperDirEnv+"="+dir, ) diff --git a/internal/metrics/BUILD.bazel b/internal/metrics/BUILD.bazel index e7d9e14c1..dd2301a52 100644 --- a/internal/metrics/BUILD.bazel +++ b/internal/metrics/BUILD.bazel @@ -33,5 +33,8 @@ go_test( "userconfig_test.go", ], embed = [":metrics"], - deps = ["@in_gopkg_yaml_v3//:yaml_v3"], + deps = [ + "//internal/testutil/bazeltest", + "@in_gopkg_yaml_v3//:yaml_v3", + ], ) diff --git a/internal/metrics/spawn_reap_test.go b/internal/metrics/spawn_reap_test.go index f29fad0e5..323f88388 100644 --- a/internal/metrics/spawn_reap_test.go +++ b/internal/metrics/spawn_reap_test.go @@ -8,6 +8,8 @@ import ( "strings" "testing" "time" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // TestStartDetachedReapsExitedChild is the GH#5900 regression: a child that @@ -27,7 +29,7 @@ func TestStartDetachedReapsExitedChild(t *testing.T) { } cmd := exec.Command(os.Args[0], "-test.run=^TestStartDetachedReapsExitedChild$") //nolint:gosec // test binary re-exec - cmd.Env = append(os.Environ(), "BD_TEST_FLUSHER_HELPER=1") + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "BD_TEST_FLUSHER_HELPER=1") cmd.Stdin = nil cmd.Stdout = nil cmd.Stderr = nil diff --git a/internal/notion/BUILD.bazel b/internal/notion/BUILD.bazel index 925e24004..beb8b2aea 100644 --- a/internal/notion/BUILD.bazel +++ b/internal/notion/BUILD.bazel @@ -23,6 +23,7 @@ go_library( go_test( name = "notion_test", srcs = [ + "auth_test.go", "client_test.go", "mapping_test.go", "refs_test.go", @@ -30,6 +31,7 @@ go_test( ], embed = [":notion"], deps = [ + "//internal/config", "//internal/tracker", "//internal/types", ], diff --git a/internal/routing/BUILD.bazel b/internal/routing/BUILD.bazel index 5b24e7fcd..827dc8522 100644 --- a/internal/routing/BUILD.bazel +++ b/internal/routing/BUILD.bazel @@ -5,12 +5,18 @@ go_library( srcs = ["routing.go"], importpath = "github.com/steveyegge/beads/internal/routing", visibility = ["//:__subpackages__"], - deps = ["//internal/git"], + deps = [ + "//internal/git", + "//internal/gitenv", + ], ) go_test( name = "routing_test", srcs = ["routing_test.go"], embed = [":routing"], - deps = ["//internal/git"], + deps = [ + "//internal/git", + "//internal/gitenv", + ], ) diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index aea9eeb3d..ad5df66e8 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -54,6 +54,7 @@ go_library( "metadata.go", "metadata_cas.go", "remote.go", + "remote_ref.go", "storage.go", "sync.go", "version_control.go", @@ -98,6 +99,7 @@ go_test( "metadata_jsonpath_test.go", "metadata_schema_test.go", "metadata_test.go", + "remote_ref_test.go", "role_accessor_decorator_test.go", ], data = [ diff --git a/internal/storage/dbproxy/proxy/BUILD.bazel b/internal/storage/dbproxy/proxy/BUILD.bazel index b6bf2cf4c..afdd198ed 100644 --- a/internal/storage/dbproxy/proxy/BUILD.bazel +++ b/internal/storage/dbproxy/proxy/BUILD.bazel @@ -81,7 +81,15 @@ go_test( "//internal/storage/dbproxy/util", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - ], + ] + select({ + "@rules_go//go/platform:android": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:linux": [ + "//internal/testutil/bazeltest", + ], + "//conditions:default": [], + }), ) # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/internal/storage/dbproxy/proxy/force_stop_linux_test.go b/internal/storage/dbproxy/proxy/force_stop_linux_test.go index 4a4f4bf83..6bec80e60 100644 --- a/internal/storage/dbproxy/proxy/force_stop_linux_test.go +++ b/internal/storage/dbproxy/proxy/force_stop_linux_test.go @@ -14,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/storage/dbproxy/pidfile" "github.com/steveyegge/beads/internal/storage/dbproxy/server" "github.com/steveyegge/beads/internal/storage/dbproxy/util" + "github.com/steveyegge/beads/internal/testutil/bazeltest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -197,7 +198,7 @@ func startNamedForceStopHelper(t *testing.T, dir, name string) helperProcess { } cmd := exec.Command(executable, "-test.run=^TestForceStopHelperProcess$") - cmd.Env = append(os.Environ(), forceStopHelperEnv+"=1") + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), forceStopHelperEnv+"=1") require.NoError(t, cmd.Start()) token, err := procid.Capture(cmd.Process.Pid) require.NoError(t, err) diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 7576e5f0c..5bf6acf98 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -140,6 +140,7 @@ go_test( "claim_verify_test.go", "claim_wrapper_smoke_test.go", "claimer_contract_test.go", + "cli_mirror_ref_test.go", "cli_transfer_timeout_test.go", "close_issue_checked_test.go", "close_policy_gate_test.go", @@ -359,7 +360,15 @@ go_test( "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_google_uuid//:uuid", - ], + ] + select({ + "@rules_go//go/platform:android": [ + "//internal/testutil/bazeltest", + ], + "@rules_go//go/platform:linux": [ + "//internal/testutil/bazeltest", + ], + "//conditions:default": [], + }), ) # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/internal/storage/dolt/suite_leak_linux_test.go b/internal/storage/dolt/suite_leak_linux_test.go index 538424153..769722d0f 100644 --- a/internal/storage/dolt/suite_leak_linux_test.go +++ b/internal/storage/dolt/suite_leak_linux_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // This process boundary exercises the real TestMain allocation and exit policy, @@ -75,6 +76,7 @@ func TestSuiteFixtureLeakFailsOwningSuite(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() child := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestSuiteFixtureLeakFailsOwningSuite$", "-test.count=1") + child.Env = bazeltest.ShardFreeEnv(os.Environ()) child.ExtraFiles = []*os.File{readEnd} out, err := child.CombinedOutput() var exitErr *exec.ExitError diff --git a/internal/storage/doltutil/BUILD.bazel b/internal/storage/doltutil/BUILD.bazel index 613bd9d24..06c0be38a 100644 --- a/internal/storage/doltutil/BUILD.bazel +++ b/internal/storage/doltutil/BUILD.bazel @@ -6,6 +6,9 @@ go_library( "close.go", "dsn.go", "identifier.go", + "probe_pid_other.go", + "probe_pid_unix.go", + "probe_pid_windows.go", "remotes.go", ], importpath = "github.com/steveyegge/beads/internal/storage/doltutil", @@ -15,12 +18,18 @@ go_library( "//internal/remotecache", "//internal/storage", "@com_github_go_sql_driver_mysql//:mysql", - ], + ] + select({ + "@rules_go//go/platform:windows": [ + "@org_golang_x_sys//windows", + ], + "//conditions:default": [], + }), ) go_test( name = "doltutil_test", srcs = [ + "cli_mirror_ref_test.go", "close_test.go", "dsn_test.go", "identifier_test.go", @@ -28,6 +37,7 @@ go_test( "remotes_test.go", ], embed = [":doltutil"], + deps = ["//internal/storage"], ) # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/internal/storage/domain/BUILD.bazel b/internal/storage/domain/BUILD.bazel index f62f1f4ba..aa6fa9091 100644 --- a/internal/storage/domain/BUILD.bazel +++ b/internal/storage/domain/BUILD.bazel @@ -28,6 +28,7 @@ go_library( "//internal/idgen", "//internal/storage", "//internal/storage/dberrors", + "//internal/storage/doltutil", "//internal/types", "//internal/validation", "//issueops", @@ -41,6 +42,7 @@ go_test( "context_test.go", "infra_types_test.go", "issue_delete_missing_table_test.go", + "remote_test.go", ], embed = [":domain"], deps = [ diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index de42aa02a..fe51e3993 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "db", @@ -94,6 +95,7 @@ go_test( "ready_work_counts_test.go", "ready_work_missing_table_test.go", "ready_work_test.go", + "remote_test.go", "reopen_test.go", "reopen_usecase_test.go", "suite_test.go", @@ -121,6 +123,30 @@ go_test( ], ) +# The Dolt-backed db suite (TestDomainDB) as pr.yml's "Test (storage domain + +# uow)" job runs it: the suite starts a dolt sql-server container through +# testcontainers, and BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 turns a missing +# container into a failure instead of a silent skip. db_test itself stays +# untagged: under --config=prcore (BEADS_TEST_SKIP=dolt) the suite skips, as in +# pr-core's `go test`. +# +# Tags: +# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, +# selected by --config=docker. +# no-remote-exec: the daemon is on the host, not on remote workers. +sh_test( + name = "db_docker_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :db_test)"], + data = [":db_test"], + env = {"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"}, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/domain/git/BUILD.bazel b/internal/storage/domain/git/BUILD.bazel index 5bcaebc89..32a3ada9b 100644 --- a/internal/storage/domain/git/BUILD.bazel +++ b/internal/storage/domain/git/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//:__subpackages__"], deps = [ "//internal/git", + "//internal/gitenv", "//internal/storage/domain", ], ) @@ -19,7 +20,9 @@ go_test( ], embed = [":git"], deps = [ + "//internal/gitenv", "//internal/storage/domain", + "@com_github_stretchr_testify//require", "@com_github_stretchr_testify//suite", ], ) diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index ee79d758a..70e52e4cf 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -90,7 +90,8 @@ go_library( go_test( name = "embeddeddolt_test", - # -race -short runs ~6 min (pr-core gives go test -timeout=30m). + # -race -short runs ~6 min unsharded (pr-core gives go test -timeout=30m). + # Bazel shards by test count, not duration. size = "large", timeout = "long", srcs = [ @@ -136,6 +137,7 @@ go_test( "external_dependencies_test.go", "federation_credentials_test.go", "get_issue_test.go", + "git_ref_pin_test.go", "graph_counter_contract_test.go", "history_presence_contract_test.go", "history_test.go", @@ -189,6 +191,7 @@ go_test( "reject_stale_upsert_test.go", "relations_contract_test.go", "releaser_contract_test.go", + "remote_git_ref_test.go", "remote_migrate_gate_test.go", "remote_peer_auth_test.go", "remote_sync_gate_test.go", @@ -212,6 +215,7 @@ go_test( "workspace_config_contract_test.go", ], embed = [":embeddeddolt"], + shard_count = 8, deps = [ "//:beads", "//backend/conformance", @@ -221,16 +225,20 @@ go_test( "//internal/storage", "//internal/storage/dberrors", "//internal/storage/depid", + "//internal/storage/doltutil", "//internal/storage/domain/db", "//internal/storage/externaldeps", "//internal/storage/issueops", "//internal/storage/schema", "//internal/storage/storagecontract", "//internal/storage/versioncontrolops", + "//internal/testutil/bazeltest", "//internal/types", "//internal/workapi", "//issueops", "@com_github_data_dog_go_sqlmock//:go-sqlmock", + "@com_github_dolthub_dolt_go//libraries/doltcore/dbfactory", + "@com_github_dolthub_dolt_go//store/blobstore", "@com_github_google_uuid//:uuid", "@org_golang_x_sync//errgroup", ], diff --git a/internal/storage/embeddeddolt/concurrency_test.go b/internal/storage/embeddeddolt/concurrency_test.go index 40c0c304c..4c8855ad6 100644 --- a/internal/storage/embeddeddolt/concurrency_test.go +++ b/internal/storage/embeddeddolt/concurrency_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/testutil/bazeltest" "golang.org/x/sync/errgroup" ) @@ -96,7 +97,9 @@ func TestConcurrencyMultiProcess(t *testing.T) { eg.Go(func() error { t.Logf("proc %d: starting", procN) cmd := exec.CommandContext(egCtx, testBin, "-test.run=^TestHelperProcess$", "-test.v") - cmd.Env = append(os.Environ(), + // ShardFreeEnv: a sharded parent's TEST_SHARD_* would filter the + // child's one -test.run test away. + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "BEADS_EMBEDDED_DOLT_HELPER=1", "BEADS_EMBEDDED_DOLT_DIR="+sharedDir, "BEADS_EMBEDDED_DOLT_ITERS="+strconv.Itoa(iters), diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index f248905e7..0862a9118 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "uow", @@ -184,6 +185,37 @@ go_test( ], ) +# The container-backed uow tests as pr.yml's "Test (storage domain + uow)" job +# runs them (no -short; each test starts its own dolt sql-server container), +# with BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 so a missing container fails instead +# of skipping. It repeats uow_test's `env` (go_test_variant.sh cannot inherit +# it). uow_test itself stays untagged: under --config=prcore those tests skip. +# +# Tags: +# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, +# selected by --config=docker. +# no-remote-exec: the daemon is on the host, not on remote workers. +sh_test( + name = "uow_docker_test", + size = "large", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :uow_test)"], + data = [ + ":uow_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, + # ~12 min unsharded under -race on a loaded host (large allows 15). + shard_count = 4, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/versioncontrolops/BUILD.bazel b/internal/storage/versioncontrolops/BUILD.bazel index 79dd60aa2..d33ea0127 100644 --- a/internal/storage/versioncontrolops/BUILD.bazel +++ b/internal/storage/versioncontrolops/BUILD.bazel @@ -44,6 +44,7 @@ go_test( "gc_test.go", "remotes_githooks_test.go", "remotes_gittrace_test.go", + "remotes_test.go", ], embed = [":versioncontrolops"], deps = [ diff --git a/internal/testutil/testdoltserver.go b/internal/testutil/testdoltserver.go index 2831231e7..367a0be70 100644 --- a/internal/testutil/testdoltserver.go +++ b/internal/testutil/testdoltserver.go @@ -98,6 +98,21 @@ func hasTestSkip(service string) bool { return false } +// EnvRequireDoltContainer, set to "1", turns a missing Dolt test container into +// a test failure instead of a skip in RequireDoltContainer and +// StartIsolatedDoltContainer(Handle). Lanes that exist to run the container +// suites (Bazel's requires-docker variants) set it so they cannot pass green +// having run nothing. +const EnvRequireDoltContainer = "BEADS_TEST_REQUIRE_DOLT_CONTAINER" + +func skipOrFailDoltUnavailable(t *testing.T, state doltReadiness) { + t.Helper() + if os.Getenv(EnvRequireDoltContainer) == "1" { + t.Fatalf("Dolt test container unavailable (%s) but %s=1; this lane must not skip", state, EnvRequireDoltContainer) + } + t.Skipf("skipping test: %s", state) +} + // checkDolt returns the readiness state for Dolt integration tests. // It composes hasTestSkip, isDockerAvailable, isDoltImageCached, and // isDoltRepoImageCached, caching the result. @@ -349,7 +364,7 @@ func (c *IsolatedDoltContainer) Exec(ctx context.Context, cmd []string) (int, st func StartIsolatedDoltContainerHandle(t *testing.T) *IsolatedDoltContainer { t.Helper() if state := checkDolt(); state != doltReady { - t.Skipf("skipping test: %s", state) + skipOrFailDoltUnavailable(t, state) } ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) @@ -420,7 +435,7 @@ func EnsureDoltContainerForTestMain() error { func RequireDoltContainer(t *testing.T) { t.Helper() if state := checkDolt(); state != doltReady { - t.Skipf("skipping test: %s", state) + skipOrFailDoltUnavailable(t, state) } ensureSharedContainer() diff --git a/internal/workspacegate/BUILD.bazel b/internal/workspacegate/BUILD.bazel index 54fbdad7d..530171afb 100644 --- a/internal/workspacegate/BUILD.bazel +++ b/internal/workspacegate/BUILD.bazel @@ -12,4 +12,5 @@ go_test( name = "workspacegate_test", srcs = ["gate_test.go"], embed = [":workspacegate"], + deps = ["//internal/testutil/bazeltest"], ) diff --git a/internal/workspacegate/gate_test.go b/internal/workspacegate/gate_test.go index cdeefe39e..4bcdd3ff1 100644 --- a/internal/workspacegate/gate_test.go +++ b/internal/workspacegate/gate_test.go @@ -14,6 +14,8 @@ import ( "strings" "testing" "time" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // TestMain doubles as the cross-process helper: when WORKSPACEGATE_HELPER @@ -477,7 +479,7 @@ func deadPID(t *testing.T) int { t.Fatalf("os.Executable: %v", err) } cmd := exec.Command(exe) - cmd.Env = append(os.Environ(), "WORKSPACEGATE_HELPER=exit") + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "WORKSPACEGATE_HELPER=exit") if err := cmd.Run(); err != nil { t.Fatalf("running exit helper: %v", err) } @@ -599,7 +601,7 @@ func spawnHolder(t *testing.T, dir, mode string) *holderProc { t.Fatalf("os.Executable: %v", err) } cmd := exec.Command(exe, dir, mode) - cmd.Env = append(os.Environ(), "WORKSPACEGATE_HELPER=hold") + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "WORKSPACEGATE_HELPER=hold") stdin, err := cmd.StdinPipe() if err != nil { t.Fatal(err) @@ -710,7 +712,7 @@ func TestSpawnedChildDoesNotInheritGate(t *testing.T) { t.Fatal(err) } child := exec.Command(exe) - child.Env = append(os.Environ(), "WORKSPACEGATE_HELPER=sleep") + child.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "WORKSPACEGATE_HELPER=sleep") if err := child.Start(); err != nil { t.Fatal(err) } diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 225817e09..934558474 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -1,5 +1,19 @@ load("@rules_go//go:def.bzl", "go_test") +# CI and repo-policy tests: they parse the workflows and run the CI shell +# scripts against fixtures. +# +# Tags: +# host-tools: the scripts and fixtures run the host's git, python3, awk, +# sed, find, grep and timeout (beyond the bash the test wrapper needs). +# Under Bazel a missing git or bash fails the test (under go test it +# skips). +# no-remote-exec: so they always run where that inventory is known (the +# developer machine or the CI runner). The farm workers do have these +# tools today (the target passes there), so this can be dropped once that +# is guaranteed. +# no-remote-cache: the action key does not cover the host's tools, so a +# result from one host is neither uploaded for nor served to another. go_test( name = "scripts_test", srcs = [ @@ -11,19 +25,25 @@ go_test( "ci_workflow_test.go", "complexity_script_test.go", "conformance_script_test.go", + "host_tools_test.go", "install_dolt_test.go", "pull_dolt_image_test.go", "release_script_test.go", "test_script_test.go", ], data = [ - "ci/install-dolt.sh", - "ci/pull-dolt-image.sh", + ":scripts_files", "//:bazel_policy_files", + "//:test_sh_files", "//:workflow_files", "//internal/testutil:testdoltcommon.go", "//tools/bazel:dolt.bzl", ], + tags = [ + "host-tools", + "no-remote-cache", + "no-remote-exec", + ], deps = [ "//internal/formula", "//internal/testutil/bazeltest", @@ -31,11 +51,30 @@ go_test( ], ) +# The scripts scripts_test reads and runs (CI helpers, check-*.sh, release.sh, +# test.sh, conformance.sh, ...). Globs stop at subpackages, so this is only +# this package's own tree. +filegroup( + name = "scripts_files", + srcs = glob( + ["**"], + exclude = [ + "BUILD.bazel", + "**/*_test.go", + ], + ), +) + exports_files( ["migration-test/release-catalog.json"], visibility = ["//scripts/migration-test/catalog:__pkg__"], ) +exports_files( + ["ci/fmt-check.sh"], + visibility = ["//scripts/prlintmake:__pkg__"], +) + exports_files( ["README.md"], visibility = ["//:__pkg__"], diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index b0247cd6d..a674583d5 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -663,3 +663,225 @@ func TestBazelNoPackagesUnderDocsTrees(t *testing.T) { } } } + +// --- test tag taxonomy --------------------------------------------------------- + +// allowedBazelTestTags is the tag taxonomy. A target with no tags is hermetic +// and runs everywhere (local, remote, PR). Every other tag must be one of +// these and must be justified in a comment right above the rule that uses it. +var allowedBazelTestTags = map[string]string{ + "requires-dolt": "uses the hermetic pinned dolt CLI (informational)", + "host-tools": "needs host tools (bash/git/make/jq/...) beyond the test wrapper's", + "no-remote-exec": "must run on the Bazel client's host, never on a remote worker", + "no-remote-cache": "result depends on the host, so it is neither read from nor uploaded to the remote cache", + "requires-docker": "needs a docker daemon; excluded from --config=prcore/ci, run by --config=docker", + "embedded": "embedded-Dolt tier variant; its own config", + "manual": "repro/bench harness; never part of //...", +} + +// bazelTagsRequiring maps tags whose targets depend on the host to the tags +// they must also carry: no-remote-exec, or remote execution would run them on a +// worker that lacks the tool or daemon; and for host-tools, no-remote-cache, +// because the action key does not cover the host's tool inventory, so a result +// produced on one host must not be served to another. (requires-docker +// targets fail rather than skip without their daemon, so their passes are +// safe to share.) +var bazelTagsRequiring = map[string][]string{ + "host-tools": {"no-remote-exec", "no-remote-cache"}, + "requires-docker": {"no-remote-exec"}, +} + +var ( + bazelTagsAttrRe = regexp.MustCompile(`\btags\s*=\s*`) + bazelTopRuleRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_.]*\(`) + bazelQuotedStrRe = regexp.MustCompile(`"([^"]*)"`) +) + +// checkBazelBuildTags checks every `tags = [...]` in one BUILD file: the list +// is a literal, each tag is in the taxonomy and is named in the comment block +// directly above its rule, and host-bound tags come with no-remote-exec. +func checkBazelBuildTags(name, build string) []error { + var errs []error + lines := strings.Split(build, "\n") + for _, loc := range bazelTagsAttrRe.FindAllStringIndex(build, -1) { + line := strings.Count(build[:loc[0]], "\n") + rest := build[loc[1]:] + if !strings.HasPrefix(rest, "[") { + errs = append(errs, errors.New(name+":"+strconv.Itoa(line+1)+": tags must be a literal list")) + continue + } + end := strings.Index(rest, "]") + if end < 0 { + errs = append(errs, errors.New(name+":"+strconv.Itoa(line+1)+": unterminated tags list")) + continue + } + // `tags = ["x"] + OTHER` is not a literal either: only a comma, the + // rule's closing paren or the end of the line may follow the list. + if after := strings.TrimLeft(rest[end+1:], " \t"); after != "" && !strings.ContainsAny(after[:1], ",)\n#") { + errs = append(errs, errors.New(name+":"+strconv.Itoa(line+1)+": tags must be a literal list")) + continue + } + var tags []string + for _, m := range bazelQuotedStrRe.FindAllStringSubmatch(rest[:end], -1) { + tags = append(tags, m[1]) + } + + // The rule is the nearest top-level call above; its justification is + // the contiguous comment block right above that. + start := line + for start > 0 && !bazelTopRuleRe.MatchString(lines[start]) { + start-- + } + var comment strings.Builder + for i := start - 1; i >= 0 && strings.HasPrefix(strings.TrimSpace(lines[i]), "#"); i-- { + comment.WriteString(lines[i]) + comment.WriteByte('\n') + } + where := name + ":" + strconv.Itoa(start+1) + + have := map[string]bool{} + for _, tag := range tags { + have[tag] = true + if _, ok := allowedBazelTestTags[tag]; !ok { + errs = append(errs, errors.New(where+": tag "+strconv.Quote(tag)+" is not in the taxonomy (allowedBazelTestTags)")) + continue + } + if !strings.Contains(comment.String(), tag) { + errs = append(errs, errors.New(where+": tag "+strconv.Quote(tag)+" is not justified in a comment directly above the rule")) + } + } + for _, tag := range tags { + for _, need := range bazelTagsRequiring[tag] { + if !have[need] { + errs = append(errs, errors.New(where+": tag "+strconv.Quote(tag)+" also requires "+need)) + } + } + } + } + return errs +} + +func TestBazelTestTagsFollowTaxonomy(t *testing.T) { + for name, good := range map[string]string{ + "docker": "# Tags:\n# requires-docker: needs a daemon.\n# no-remote-exec: the daemon is local.\n" + + "sh_test(\n name = \"x\",\n tags = [\n \"no-remote-exec\",\n \"requires-docker\",\n ],\n)\n", + "host-tools": "# host-tools, no-remote-exec, no-remote-cache: git.\n" + + "go_test(\n name = \"x\",\n tags = [\"host-tools\", \"no-remote-exec\", \"no-remote-cache\"], # why\n)\n", + } { + if errs := checkBazelBuildTags(name, good); len(errs) != 0 { + t.Errorf("%s: justified fixture rejected: %v", name, errs) + } + } + for name, build := range map[string]string{ + "unknown tag": "# flaky: why\ngo_test(\n name = \"x\",\n tags = [\"flaky\"],\n)\n", + "no comment": "go_test(\n name = \"x\",\n tags = [\"manual\"],\n)\n", + "comment too far": "# manual: harness\n\ngo_test(\n name = \"x\",\n tags = [\"manual\"],\n)\n", + "other rule's": "# manual: harness\ngo_test(name = \"a\", tags = [\"manual\"])\n\ngo_test(\n name = \"b\",\n tags = [\"manual\"],\n)\n", + "docker unpinned": "# requires-docker: daemon\ngo_test(\n name = \"x\",\n tags = [\"requires-docker\"],\n)\n", + "not a literal": "# manual\ngo_test(\n name = \"x\",\n tags = MANUAL,\n)\n", + "literal plus": "# manual\ngo_test(\n name = \"x\",\n tags = [\"manual\"] + MORE,\n)\n", + "host cacheable": "# host-tools no-remote-exec\ngo_test(\n name = \"x\",\n tags = [\"host-tools\", \"no-remote-exec\"],\n)\n", + "one-line rule": "go_test(name = \"x\", tags = [\"manual\"])\n", + } { + if errs := checkBazelBuildTags(name, build); len(errs) == 0 { + t.Errorf("%s: expected a tag policy error for fixture:\n%s", name, build) + } + } + + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("walks every BUILD file in the source checkout; runs under go test") + } + root := sourceRepoRoot(t) + pkgs, err := bazelPackagesUnder(root, ".") + if err != nil { + t.Fatalf("walk: %v", err) + } + if len(pkgs) == 0 { + t.Fatal("found no BUILD.bazel packages; the walk is broken") + } + for _, pkg := range append([]string{"."}, pkgs...) { + rel := filepath.ToSlash(filepath.Join(pkg, "BUILD.bazel")) + for _, err := range checkBazelBuildTags(rel, readPolicyFile(t, root, rel)) { + t.Error(err) + } + } +} + +// checkBazelrcPrcoreTagFilter requires --config=prcore to exclude the tags +// that never run in the PR-core lane. Every --test_tag_filters set by prcore or +// by a config that expands it (test:ci) is checked, since a later one would +// override the first. +func checkBazelrcPrcoreTagFilter(bazelrc string) error { + type option struct{ config, flag string } + var opts []option + for _, line := range strings.Split(bazelrc, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || strings.HasPrefix(fields[0], "#") { + continue + } + _, config, ok := strings.Cut(fields[0], ":") + if !ok { + continue + } + for i := 1; i < len(fields); i++ { + flag := fields[i] + if (flag == "--config" || flag == "--test_tag_filters") && i+1 < len(fields) { + i++ + flag += "=" + fields[i] + } + opts = append(opts, option{config, flag}) + } + } + + lane := map[string]bool{"prcore": true} + for grew := true; grew; { + grew = false + for _, o := range opts { + if !lane[o.config] && strings.HasPrefix(o.flag, "--config=") && lane[strings.TrimPrefix(o.flag, "--config=")] { + lane[o.config] = true + grew = true + } + } + } + + prcoreFilter := false + for _, o := range opts { + if !lane[o.config] || !strings.HasPrefix(o.flag, "--test_tag_filters=") { + continue + } + prcoreFilter = prcoreFilter || o.config == "prcore" + filters := map[string]bool{} + for _, f := range strings.Split(strings.TrimPrefix(o.flag, "--test_tag_filters="), ",") { + filters[f] = true + } + for _, tag := range []string{"requires-docker", "embedded", "manual"} { + if !filters["-"+tag] { + return errors.New(o.config + " --test_tag_filters does not exclude " + tag) + } + } + } + if !prcoreFilter { + return errors.New(".bazelrc has no test:prcore --test_tag_filters line") + } + return nil +} + +func TestBazelrcPrcoreExcludesNonPRTags(t *testing.T) { + if err := checkBazelrcPrcoreTagFilter(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc")); err != nil { + t.Fatal(err) + } + for name, rc := range map[string]string{ + "missing": "test:ci --keep_going\n", + "no docker": "test:prcore --test_tag_filters=-embedded,-manual\n", + "ci override": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + + "test:ci --config=prcore\ntest:ci --test_tag_filters=requires-docker\n", + "second prcore line": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + + "test:prcore --test_tag_filters=-manual\n", + "transitive": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + + "test:ci --config=prcore\nbuild:nightly --config ci --test_tag_filters=\n", + } { + if err := checkBazelrcPrcoreTagFilter(rc); err == nil { + t.Errorf("%s: expected an error for .bazelrc fixture:\n%s", name, rc) + } + } +} diff --git a/scripts/check_build_tags_test.go b/scripts/check_build_tags_test.go index 1ececfc31..e37c9cf25 100644 --- a/scripts/check_build_tags_test.go +++ b/scripts/check_build_tags_test.go @@ -17,12 +17,10 @@ func runCheckBuildTags(t *testing.T, files map[string]string) (string, error) { if runtime.GOOS == "windows" { t.Skip("checker is a Bash boundary") } - if _, err := exec.LookPath("git"); err != nil { - t.Skip("git not available") - } + requireHostTool(t, "git") // The checker needs bash >= 4 (mapfile); macOS ships bash 3.2 as /bin/bash. if err := exec.Command("bash", "-c", "type mapfile").Run(); err != nil { - t.Skip("bash lacks mapfile (bash >= 4 required)") + skipOrFailWithoutHostTool(t, "bash lacks mapfile (bash >= 4 required)") } script, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), "scripts", "check-build-tags.sh")) if err != nil { diff --git a/scripts/check_testing_short_test.go b/scripts/check_testing_short_test.go index 40ef98810..10c7a7a96 100644 --- a/scripts/check_testing_short_test.go +++ b/scripts/check_testing_short_test.go @@ -70,6 +70,11 @@ func TestCheckTestingShortPassesOnCleanRepoTree(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("checker is a Bash boundary") } + if os.Getenv("TEST_SRCDIR") != "" { + // Under Bazel the tree is only this target's runfiles, so the scan + // would pass vacuously. make check-testing-short runs it in CI. + t.Skip("walks the whole source checkout; runs under go test") + } repo := sourceRepoRoot(t) cmd := exec.Command("bash", filepath.Join(repo, "scripts", "check-testing-short.sh")) cmd.Dir = repo diff --git a/scripts/complexity_script_test.go b/scripts/complexity_script_test.go index b856de530..62080e450 100644 --- a/scripts/complexity_script_test.go +++ b/scripts/complexity_script_test.go @@ -102,7 +102,7 @@ func TestComplexityScriptDiffReportsCrossingAndTrueDeletionOnly(t *testing.T) { // The phase file is an explicit head/base discriminator. This keeps the // fixture independent of checkout directory names and worktree layout. writeExecutable(t, fake, "#!/bin/sh\nif [ \"$(cat "+shellQuote(phase)+")\" = head ]; then\nprintf '%s\\n' '31 main Crossing cmd/cross.go:2:1' '10 main Drop cmd/drop.go:2:1'\nprintf '%s\\n' base >"+shellQuote(phase)+"\nelse\nprintf '%s\\n' '25 main Crossing cmd/cross.go:1:1' '58 main Drop cmd/drop.go:1:1' '40 main Gone cmd/gone.go:1:1' '12 main Quiet cmd/quiet.go:1:1'\nprintf '%s\\n' head >"+shellQuote(phase)+"\nfi\n") - out, err := runComplexityDiff(t, sourceRepoRoot(t), fake) + out, err := runComplexityDiff(t, complexityGitFixture(t), fake) if err != nil { t.Fatalf("diff failed: %v\n%s", err, out) } @@ -157,6 +157,48 @@ func runComplexityDiff(t *testing.T, repo, tool string) (string, error) { return string(data), err } +// complexityGitFixture returns a one-commit git repository holding +// scripts/ci/complexity.sh, for diff mode (which needs COMPLEXITY_BASE_REF to +// resolve). The fake analyzer ignores the tree, so the fixture needs no Go +// source beyond one placeholder; it keeps the test off the real checkout's history, which Bazel's +// runfiles tree does not have. +func complexityGitFixture(t *testing.T) string { + t.Helper() + requireHostTool(t, "git") + script, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), "scripts", "ci", "complexity.sh")) + if err != nil { + t.Fatal(err) + } + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, "scripts", "ci"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "scripts", "ci", "complexity.sh"), script, 0o755); err != nil { + t.Fatal(err) + } + // complexity.sh scans cmd/ etc.; an empty path list trips `set -u` on + // bash 3.2 (macOS), so give it one package like the real tree has. + if err := os.MkdirAll(filepath.Join(repo, "cmd"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "cmd", "cross.go"), []byte("package main\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{ + {"init", "-q"}, + {"add", "-A"}, + {"-c", "user.name=beads-test", "-c", "user.email=test@beads.local", "-c", "commit.gpgsign=false", + "commit", "-q", "-m", "fixture"}, + } { + cmd := exec.Command("git", args...) + cmd.Dir = repo + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + return repo +} + func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" } diff --git a/scripts/conformance_script_test.go b/scripts/conformance_script_test.go index 7b920e953..2bad6f56e 100644 --- a/scripts/conformance_script_test.go +++ b/scripts/conformance_script_test.go @@ -87,10 +87,7 @@ type conformanceRun struct { func runConformanceScript(t *testing.T, failCall, failExit int) conformanceRun { t.Helper() - bash, err := exec.LookPath("bash") - if err != nil { - t.Skipf("bash is required to test conformance.sh: %v", err) - } + bash := requireHostTool(t, "bash") bin := t.TempDir() stateDir := t.TempDir() diff --git a/scripts/host_tools_test.go b/scripts/host_tools_test.go new file mode 100644 index 000000000..76b0874d3 --- /dev/null +++ b/scripts/host_tools_test.go @@ -0,0 +1,32 @@ +package scripts_test + +import ( + "os/exec" + "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +// skipOrFailWithoutHostTool handles a missing host tool. Under go test the +// test skips, as it always has. Under Bazel it fails: //scripts:scripts_test +// is tagged host-tools, which promises git, python3, bash and friends on the +// host that runs it, so a skip there would hide a host that breaks that promise +// behind a green result. +func skipOrFailWithoutHostTool(t *testing.T, format string, args ...any) { + t.Helper() + if bazeltest.IsBazel() { + t.Fatalf("host-tools target: "+format, args...) + } + t.Skipf(format, args...) +} + +// requireHostTool returns the path of a host tool, or skips/fails via +// skipOrFailWithoutHostTool when it is missing. +func requireHostTool(t *testing.T, name string) string { + t.Helper() + path, err := exec.LookPath(name) + if err != nil { + skipOrFailWithoutHostTool(t, "%s not available: %v", name, err) + } + return path +} diff --git a/scripts/install_dolt_test.go b/scripts/install_dolt_test.go index 14a828ceb..654632ec6 100644 --- a/scripts/install_dolt_test.go +++ b/scripts/install_dolt_test.go @@ -2,7 +2,6 @@ package scripts_test import ( "os" - "os/exec" "path/filepath" "runtime" "strconv" @@ -155,10 +154,7 @@ type installDoltRun struct { func runInstallDolt(t *testing.T, curlFailures int, reportedVersion string) installDoltRun { t.Helper() - bash, err := exec.LookPath("bash") - if err != nil { - t.Skipf("bash is required to test install-dolt.sh: %v", err) - } + bash := requireHostTool(t, "bash") bin := t.TempDir() stateDir := t.TempDir() diff --git a/scripts/prlintmake/BUILD.bazel b/scripts/prlintmake/BUILD.bazel index 813c6d873..fe68f5879 100644 --- a/scripts/prlintmake/BUILD.bazel +++ b/scripts/prlintmake/BUILD.bazel @@ -3,4 +3,6 @@ load("@rules_go//go:def.bzl", "go_test") go_test( name = "prlintmake_test", srcs = ["prlintmake_test.go"], + data = ["//scripts:ci/fmt-check.sh"], + deps = ["//internal/testutil/bazeltest"], ) diff --git a/scripts/prlintmake/prlintmake_test.go b/scripts/prlintmake/prlintmake_test.go index 0041f8df7..45e6bad3c 100644 --- a/scripts/prlintmake/prlintmake_test.go +++ b/scripts/prlintmake/prlintmake_test.go @@ -8,6 +8,8 @@ import ( "runtime" "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) func TestFmtCheckClean(t *testing.T) { @@ -153,7 +155,9 @@ func sourceRepoRoot() string { if !ok { panic("runtime.Caller failed") } - return filepath.Dir(filepath.Dir(filepath.Dir(file))) + // Under Bazel the caller path is workspace-relative; CallerDir rebuilds it + // under the runfiles root, which holds the declared fmt-check.sh. + return filepath.Dir(filepath.Dir(bazeltest.CallerDir(file, "scripts/prlintmake"))) } func shellVisiblePath(path string) string { diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index eb17ce8d1..ce4107cc8 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -112,10 +112,7 @@ type pullDoltRun struct { func runPullDoltImage(t *testing.T, failures int) pullDoltRun { t.Helper() - bash, err := exec.LookPath("bash") - if err != nil { - t.Skipf("bash is required to test pull-dolt-image.sh: %v", err) - } + bash := requireHostTool(t, "bash") bin := t.TempDir() stateDir := t.TempDir() diff --git a/scripts/release_script_test.go b/scripts/release_script_test.go index 709649068..89a811de1 100644 --- a/scripts/release_script_test.go +++ b/scripts/release_script_test.go @@ -85,6 +85,7 @@ echo '{"source":"/tmp/stale.formula.toml"}' } func TestReleaseFormulaCleanupStaleDoltOrphansHandlesLocalModeWithoutJQ(t *testing.T) { + skipReleaseFormulaUnderBazel(t) repoRoot := sourceRepoRoot(t) formulaPath := filepath.Join(repoRoot, ".beads", "formulas", "beads-release.formula.toml") if _, err := formula.NewParser().ParseFile(formulaPath); err != nil { @@ -118,6 +119,7 @@ func TestReleaseFormulaCleanupStaleDoltOrphansHandlesLocalModeWithoutJQ(t *testi } func TestReleaseFormulaHomebrewCoreProcedureCoversTemplateAndBottles(t *testing.T) { + skipReleaseFormulaUnderBazel(t) repoRoot := sourceRepoRoot(t) formulaPath := filepath.Join(repoRoot, ".beads", "formulas", "beads-release.formula.toml") if _, err := formula.NewParser().ParseFile(formulaPath); err != nil { @@ -204,6 +206,17 @@ func releaseTestTempDir(t *testing.T) string { return t.TempDir() } +// skipReleaseFormulaUnderBazel skips tests that read +// .beads/formulas/beads-release.formula.toml under Bazel: .beads holds live +// Dolt data and is in .bazelignore, so no target can declare the formula. +// These tests run under `go test`. +func skipReleaseFormulaUnderBazel(t *testing.T) { + t.Helper() + if bazeltest.IsBazel() { + t.Skip(".beads/ is in .bazelignore, so the release formula cannot be declared as data; runs under go test") + } +} + func sourceRepoRoot(t *testing.T) string { t.Helper() _, file, _, ok := runtime.Caller(0) diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index cb7603069..9d7afb436 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -18,6 +18,13 @@ exports_files( visibility = ["//scripts:__pkg__"], ) +# sh_test src for a second test target over an existing go_test binary (see +# the script header), e.g. //cmd/bd/doctor/fix:fix_docker_test. +exports_files( + ["go_test_variant.sh"], + visibility = ["//visibility:public"], +) + # Wraps every test process (`test --run_under` in .bazelrc) with the isolated # HOME/XDG/Dolt/git environment that scripts/ci/lib/test-env.sh gives `go test`. # Bazel merges the run_under target's runfiles into every test's runfiles, so diff --git a/tools/bazel/go_test_variant.sh b/tools/bazel/go_test_variant.sh new file mode 100755 index 000000000..80bb135c9 --- /dev/null +++ b/tools/bazel/go_test_variant.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Runs an existing go_test binary as another test target, so a package can be +# tested a second way (different env, tags, or lane) without compiling it +# again. The first argument is the binary's runfiles path ($(rootpath ...), +# relative to the workspace runfiles directory, which is the test's working +# directory); the rest, plus any --test_arg, go to the binary unchanged. +# +# Sharding, XML output and timeouts pass through (the binary reads Bazel's +# TEST_* environment). What does NOT pass through is the go_test's own +# RunEnvironmentInfo: rules_go sets GO_TEST_RUN_FROM_BAZEL=1, which this script +# exports so the binary still changes into its package directory, and the +# go_test's `env` attribute, which the variant target must repeat in its own +# `env` (with the referenced targets in its `data`). +set -euo pipefail +bin="$1" +shift +export GO_TEST_RUN_FROM_BAZEL=1 +exec "./$bin" "$@" diff --git a/tools/bazel/test_env.sh b/tools/bazel/test_env.sh index 34059450b..fec7b5313 100755 --- a/tools/bazel/test_env.sh +++ b/tools/bazel/test_env.sh @@ -37,6 +37,28 @@ export GIT_CONFIG_GLOBAL="$root/gitconfig" export TMPDIR="$root/tmp" export BEADS_TEST_IGNORE_REPO_CONFIG=1 +# Discovery ceilings. bd walks up from its working directory for .beads and +# .beads/config.yaml, and git walks up for a repository. A test's working +# directory is in the runfiles tree under the output base, which is usually +# below the developer's HOME, so an unbounded walk would read and write their +# real ~/.beads (and a host config would decide test results). Every walk stops +# below the runfiles root, TEST_TMPDIR and this wrapper's root; a test's own +# directories under them stay discoverable. Relying on the ceiling rather than +# failing on an ancestor .beads is deliberate: a live ~/.beads above the output +# base is normal on a developer machine. +ceilings="" +for d in "${TEST_SRCDIR:-}" "${TEST_TMPDIR:-}" "$root"; do + if [[ -n "$d" && -d "$d" ]]; then + ceilings="${ceilings:+$ceilings:}$(cd "$d" && pwd -P)" + fi +done +export BEADS_CEILING_DIRECTORIES="$ceilings" +export GIT_CEILING_DIRECTORIES="$ceilings" +# The migration-freeze marker walk is deliberately not bounded by the ceiling; +# point it at a path that never exists so a MIGRATION-FREEZE file above the +# output base cannot make write commands refuse inside tests. +export BD_MIGRATION_FREEZE_FILE="$root/no-freeze-marker" + # Same scrub as beads_test_env_enter; `--test_env=NAME` on a command line must # not be able to point a test at a live workspace or Dolt server. unset BEADS_DIR BEADS_DB BD_DB BD_JSON BD_NO_DB BD_NO_DAEMON BD_ACTOR \