From c5c1fce565b2f38d0911d2e8bacc57acc4c78f37 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 18:48:30 +0000 Subject: [PATCH 001/151] perf(git): locate the repository in-process instead of running git rev-parse Every bd invocation resolved the process-wide git context (beads directory discovery reads it) with `git rev-parse --git-dir --git-common-dir --show-toplevel`: one git subprocess per bd call before any work. Tools that call bd hundreds of times (gascity's doctor issues ~250 bd calls per run) paid it on every call. On Linux, initGitContext now first runs discoverGitInProcess, which mirrors Git's discovery walk (physical cwd, .git as directory or gitfile, bare-dir check, filesystem-boundary stop) and produces byte-for-byte what rev-parse prints for ordinary repositories (top and subdirectories), linked worktrees (absolute or relative gitfile, symlinked targets), submodules (core.worktree pointing back at the gitfile's directory), nested repositories and no repository. Both answers feed one shared normalization (gitContextFromRevParse), so downstream values are unchanged. It declines, and git runs exactly as before, whenever it cannot prove the answer: GIT_DIR / GIT_WORK_TREE / GIT_COMMON_DIR / GIT_CEILING_DIRECTORIES / GIT_DISCOVERY_ACROSS_FILESYSTEM / GIT_OBJECT_DIRECTORY / GIT_CONFIG_PARAMETERS / GIT_CONFIG_COUNT in the environment; a symlinked .git, an invalid gitfile or git directory, a cwd inside a git directory; repository config with core.worktree (other than the submodule shape), core.bare, extensions, include directives, a format version above 1 or unmodeled syntax; any path not owned by the effective uid (safe.directory is git's to evaluate), or root under sudo. darwin and windows keep the subprocess (case folding and ownership semantics). Tests compare discoverGitFrom with real `git rev-parse` output across those layouts, pin every decline case, and check the resulting cached context equals the subprocess-derived one. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/git/BUILD.bazel | 7 +- internal/git/discover_linux.go | 371 ++++++++++++++++++++++++++++ internal/git/discover_linux_test.go | 247 ++++++++++++++++++ internal/git/discover_other.go | 10 + internal/git/gitdir.go | 46 +++- 5 files changed, 677 insertions(+), 4 deletions(-) create mode 100644 internal/git/discover_linux.go create mode 100644 internal/git/discover_linux_test.go create mode 100644 internal/git/discover_other.go diff --git a/internal/git/BUILD.bazel b/internal/git/BUILD.bazel index 4d259cd16..3c5b1e305 100644 --- a/internal/git/BUILD.bazel +++ b/internal/git/BUILD.bazel @@ -2,7 +2,11 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "git", - srcs = ["gitdir.go"], + srcs = [ + "discover_linux.go", + "discover_other.go", + "gitdir.go", + ], importpath = "github.com/steveyegge/beads/internal/git", visibility = ["//:__subpackages__"], ) @@ -10,6 +14,7 @@ go_library( go_test( name = "git_test", srcs = [ + "discover_linux_test.go", "gitdir_test.go", "jujutsu_test.go", ], diff --git a/internal/git/discover_linux.go b/internal/git/discover_linux.go new file mode 100644 index 000000000..7ffa7af33 --- /dev/null +++ b/internal/git/discover_linux.go @@ -0,0 +1,371 @@ +//go:build linux + +package git + +import ( + "bufio" + "bytes" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" +) + +// discoveryEnvOverrides are the environment variables that change how Git +// finds or validates a repository. When any is present, discovery is left to +// git itself. The list is deliberately broader than strictly necessary: a +// fallback costs one subprocess, a wrong answer costs a wrong workspace. +var discoveryEnvOverrides = []string{ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_CEILING_DIRECTORIES", + "GIT_DISCOVERY_ACROSS_FILESYSTEM", + "GIT_OBJECT_DIRECTORY", + "GIT_CONFIG_PARAMETERS", + "GIT_CONFIG_COUNT", + "GIT_TEST_ASSUME_DIFFERENT_OWNER", +} + +// discoverGitInProcess answers `git rev-parse --git-dir --git-common-dir +// --show-toplevel` for the process working directory without running git. ok +// is false whenever the layout is outside what it reproduces exactly; the +// caller then runs git. +func discoverGitInProcess() (revParseResult, bool) { + for _, name := range discoveryEnvOverrides { + if _, set := os.LookupEnv(name); set { + return revParseResult{}, false + } + } + wd, err := os.Getwd() + if err != nil { + return revParseResult{}, false + } + return discoverGitFrom(wd) +} + +// discoverGitFrom mirrors setup_git_directory_gently's discovery walk (Git +// 2.x) from cwd. Git walks the physical directory (getcwd), checks each +// directory's .git as a gitfile or a git directory, then the directory itself +// as a bare repository, and stops at the first filesystem boundary. +func discoverGitFrom(cwd string) (revParseResult, bool) { + cwd, err := filepath.EvalSymlinks(cwd) + if err != nil || !filepath.IsAbs(cwd) { + return revParseResult{}, false + } + cwd = filepath.Clean(cwd) + cwdStat := statOf(cwd) + if cwdStat == nil { + return revParseResult{}, false + } + + for dir := cwd; ; { + dotGit := filepath.Join(dir, ".git") + info, err := os.Lstat(dotGit) + switch { + case err == nil && info.Mode().IsDir(): + return discoveredGitDir(cwd, dir, dotGit) + case err == nil && info.Mode().IsRegular(): + return discoveredGitFile(dir, dotGit) + case err == nil: + // Symlinked .git, sockets, ...: let git decide. + return revParseResult{}, false + case !os.IsNotExist(err): + return revParseResult{}, false + } + // The directory itself may be a git directory (a bare repository, or + // somewhere inside a .git directory); git answers those differently. + if looksLikeGitDir(dir) { + return revParseResult{}, false + } + + parent := filepath.Dir(dir) + if parent == dir { + return revParseResult{notRepo: true}, true + } + parentStat := statOf(parent) + if parentStat == nil { + return revParseResult{}, false + } + if parentStat.Dev != cwdStat.Dev { + // Git stops at a mount point unless + // GIT_DISCOVERY_ACROSS_FILESYSTEM is set (handled by the caller). + return revParseResult{notRepo: true}, true + } + dir = parent + } +} + +// discoveredGitDir handles /.git being a directory. +func discoveredGitDir(cwd, top, gitDir string) (revParseResult, bool) { + // A .git directory carrying a commondir file is unusual; leave it to git. + if exists(filepath.Join(gitDir, "commondir")) { + return revParseResult{}, false + } + if !isGitDirectory(gitDir, gitDir) { + // Git would keep walking upward past an invalid .git directory. + return revParseResult{}, false + } + if !ownedByCurrentUser(top) || !ownedByCurrentUser(gitDir) { + return revParseResult{}, false + } + if worktree, ok := repoConfigAllowsDiscovery(gitDir); !ok || worktree != "" { + return revParseResult{}, false + } + if cwd == top { + return revParseResult{gitDir: ".git", commonDir: ".git", topLevel: top}, true + } + // From a subdirectory Git prints the git directory absolute and the common + // directory relative to the working directory. + rel, err := filepath.Rel(top, cwd) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + return revParseResult{}, false + } + depth := len(strings.Split(rel, string(filepath.Separator))) + return revParseResult{ + gitDir: gitDir, + commonDir: strings.Repeat("../", depth) + ".git", + topLevel: top, + }, true +} + +// discoveredGitFile handles /.git being a gitfile ("gitdir: "), as +// in linked worktrees and submodules. +func discoveredGitFile(top, gitFile string) (revParseResult, bool) { + data, err := os.ReadFile(gitFile) // #nosec G304 -- the .git file Git itself would read during discovery + if err != nil { + return revParseResult{}, false + } + content := strings.TrimRight(string(data), "\r\n") + target, found := strings.CutPrefix(content, "gitdir: ") + if !found || target == "" || strings.ContainsAny(target, "\n\r\x00") { + return revParseResult{}, false + } + if !filepath.IsAbs(target) { + target = filepath.Join(top, target) + } + gitDir, err := filepath.EvalSymlinks(target) + if err != nil { + return revParseResult{}, false + } + commonDir := gitDir + if raw, err := os.ReadFile(filepath.Join(gitDir, "commondir")); err == nil { + common := strings.TrimRight(string(raw), "\r\n") + if common == "" || strings.ContainsAny(common, "\n\r\x00") { + return revParseResult{}, false + } + if !filepath.IsAbs(common) { + common = filepath.Join(gitDir, common) + } + if commonDir, err = filepath.EvalSymlinks(common); err != nil { + return revParseResult{}, false + } + } else if !os.IsNotExist(err) { + return revParseResult{}, false + } + if !isGitDirectory(gitDir, commonDir) { + return revParseResult{}, false + } + if !ownedByCurrentUser(gitFile) || !ownedByCurrentUser(top) || !ownedByCurrentUser(gitDir) { + return revParseResult{}, false + } + worktree, ok := repoConfigAllowsDiscovery(commonDir) + if !ok { + return revParseResult{}, false + } + if worktree != "" && !submoduleWorktreeIs(gitDir, commonDir, worktree, top) { + return revParseResult{}, false + } + // Git prints both absolute from anywhere in the work tree. + return revParseResult{gitDir: gitDir, commonDir: commonDir, topLevel: top}, true +} + +// isGitDirectory mirrors Git's is_git_directory: a valid HEAD in the git +// directory, objects/ and refs/ in the common directory. +func isGitDirectory(gitDir, commonDir string) bool { + if !validHeadRef(filepath.Join(gitDir, "HEAD")) { + return false + } + return isDir(filepath.Join(commonDir, "objects")) && isDir(filepath.Join(commonDir, "refs")) +} + +// looksLikeGitDir is a cheap superset test for "git might treat dir as a git +// directory"; a hit only means discovery is handed to git. +func looksLikeGitDir(dir string) bool { + return exists(filepath.Join(dir, "HEAD")) && + (exists(filepath.Join(dir, "objects")) || exists(filepath.Join(dir, "commondir"))) +} + +// validHeadRef accepts the HEAD forms Git's validate_headref does for the +// files it writes itself: "ref: refs/..." or a full object id. Symlinked HEAD +// (an ancient layout) is left to git. +func validHeadRef(path string) bool { + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() { + return false + } + data, err := os.ReadFile(path) // #nosec G304 -- HEAD of the discovered git directory + if err != nil { + return false + } + if ref, ok := bytes.CutPrefix(data, []byte("ref:")); ok { + return bytes.HasPrefix(bytes.TrimLeft(ref, " \t"), []byte("refs/")) + } + line := strings.TrimRight(string(data), "\r\n") + if len(line) != 40 && len(line) != 64 { + return false + } + for _, c := range line { + if !strings.ContainsRune("0123456789abcdef", c) { + return false + } + } + return true +} + +// submoduleWorktreeIs reports whether a gitfile repository's core.worktree +// names exactly the directory holding the gitfile. Git writes that for every +// submodule (core.worktree = ../../../sub in .git/modules/sub/config), and Git +// then resolves the work tree by chdir(gitdir) + chdir(core.worktree), which +// lands on top again. Only that plain shape is accepted: a separate git +// directory (no shared common directory) and a value of leading "../" +// segments followed by plain names, which resolves lexically from the +// already-physical git directory exactly as the chdirs do. +func submoduleWorktreeIs(gitDir, commonDir, worktree, top string) bool { + if gitDir != commonDir || filepath.IsAbs(worktree) { + return false + } + rest := filepath.ToSlash(filepath.Clean(worktree)) + if rest != worktree && rest+"/" != worktree { + return false + } + for strings.HasPrefix(rest, "../") { + rest = rest[len("../"):] + } + for _, part := range strings.Split(rest, "/") { + if part == "" || part == "." || part == ".." { + return false + } + } + resolved, err := filepath.EvalSymlinks(filepath.Join(gitDir, worktree)) + return err == nil && resolved == top +} + +// repoConfigAllowsDiscovery reports whether the repository config leaves +// Git's answer to the plain discovery result, and returns core.worktree when +// it is set (the caller decides whether it is the submodule shape). Git +// consults only the repository's own config file here +// (read_repository_format), so global and system config cannot move the work +// tree. Anything else that could — core.bare, extensions (worktreeConfig, +// unknown ones Git rejects), a format version Git refuses, include +// directives, a repeated core.worktree, syntax this scanner does not model — +// sends discovery to git. +func repoConfigAllowsDiscovery(commonDir string) (worktree string, ok bool) { + f, err := os.Open(filepath.Join(commonDir, "config")) // #nosec G304 -- the discovered repository config + if os.IsNotExist(err) { + return "", true + } + if err != nil { + return "", false + } + defer f.Close() + + section := "" + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || line[0] == '#' || line[0] == ';' { + continue + } + if strings.HasSuffix(line, "\\") { + return "", false // continuation lines are not modeled + } + if line[0] == '[' { + end := strings.IndexByte(line, ']') + if end < 0 { + return "", false + } + header := strings.TrimSpace(line[1:end]) + if strings.ContainsAny(header, " \t\"") { + section = "" // a subsection: [remote "origin"], [branch "main"], ... + if name := strings.ToLower(strings.Fields(header)[0]); strings.HasPrefix(name, "include") { + return "", false + } + } else { + section = strings.ToLower(header) + } + if strings.HasPrefix(section, "include") || section == "extensions" { + return "", false + } + line = strings.TrimSpace(line[end+1:]) + if line == "" || line[0] == '#' || line[0] == ';' { + continue + } + } + if section != "core" { + continue + } + key, value, _ := strings.Cut(line, "=") + key = strings.ToLower(strings.TrimSpace(key)) + rawValue := value + value = strings.ToLower(strings.Trim(strings.TrimSpace(value), "\"")) + switch key { + case "worktree": + rawValue = strings.TrimSpace(rawValue) + if worktree != "" || rawValue == "" || strings.ContainsAny(rawValue, "\\\"#;") { + return "", false + } + worktree = rawValue + case "bare": + if value != "false" && value != "no" && value != "off" && value != "0" { + return "", false + } + case "repositoryformatversion": + if v, err := strconv.Atoi(value); err != nil || v < 0 || v > 1 { + return "", false + } + } + } + return worktree, scanner.Err() == nil +} + +// ownedByCurrentUser mirrors Git's is_path_owned_by_current_uid (lstat, owner +// equals the effective uid). Git's sudo allowance (root with SUDO_UID) is left +// to git. A path owned by someone else engages safe.directory, which only git +// evaluates. +func ownedByCurrentUser(path string) bool { + euid := os.Geteuid() + if euid == 0 { + if _, set := os.LookupEnv("SUDO_UID"); set { + return false + } + } + info, err := os.Lstat(path) + if err != nil { + return false + } + st, ok := info.Sys().(*syscall.Stat_t) + return ok && int(st.Uid) == euid +} + +// statOf returns path's stat data (following symlinks, as Git's +// get_device_or_die does), or nil. +func statOf(path string) *syscall.Stat_t { + info, err := os.Stat(path) + if err != nil { + return nil + } + st, _ := info.Sys().(*syscall.Stat_t) + return st +} + +func exists(path string) bool { + _, err := os.Lstat(path) + return err == nil +} + +func isDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} diff --git a/internal/git/discover_linux_test.go b/internal/git/discover_linux_test.go new file mode 100644 index 000000000..c956ba34d --- /dev/null +++ b/internal/git/discover_linux_test.go @@ -0,0 +1,247 @@ +//go:build linux + +package git + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// discoveryFixtureEnv is the environment fixture git runs under: the process +// environment minus every discovery override, with no system or user config. +func discoveryFixtureEnv(t *testing.T) []string { + t.Helper() + var env []string + for _, kv := range os.Environ() { + name, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(name, "GIT_") || name == "HOME" { + continue + } + env = append(env, kv) + } + return append(env, "HOME="+t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com") +} + +func runFixtureGit(t *testing.T, env []string, dir string, args ...string) string { + t.Helper() + cmd := gitCommand(args...) + cmd.Dir, cmd.Env = dir, env + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s in %s: %v\n%s", strings.Join(args, " "), dir, err, out) + } + return string(out) +} + +// gitRevParse is the reference answer discoverGitFrom must reproduce. +func gitRevParse(t *testing.T, env []string, dir string) revParseResult { + t.Helper() + cmd := exec.Command("git", "rev-parse", "--git-dir", "--git-common-dir", "--show-toplevel") + cmd.Dir, cmd.Env = dir, env + out, err := cmd.Output() + if err != nil { + return revParseResult{notRepo: true} + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + if len(lines) != 3 { + t.Fatalf("git rev-parse in %s: unexpected output %q", dir, out) + } + return revParseResult{gitDir: lines[0], commonDir: lines[1], topLevel: lines[2]} +} + +type discoveryFixture struct { + root, main, worktree, relWorktree, symWorktree, submodule, nested string + env []string +} + +func newDiscoveryFixture(t *testing.T) discoveryFixture { + t.Helper() + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + env := discoveryFixtureEnv(t) + f := discoveryFixture{ + root: root, + main: filepath.Join(root, "main"), + worktree: filepath.Join(root, "wt"), + relWorktree: filepath.Join(root, "wt-rel"), + symWorktree: filepath.Join(root, "wt-sym"), + env: env, + } + f.submodule = filepath.Join(f.main, "sub") + f.nested = filepath.Join(f.main, "a", "inner") + subSrc := filepath.Join(root, "subsrc") + for _, dir := range []string{f.main, subSrc, filepath.Join(f.main, "a", "b"), f.nested, filepath.Join(root, "norepo", "x")} { + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + for _, dir := range []string{f.main, subSrc} { + runFixtureGit(t, env, dir, "init", "-q") + runFixtureGit(t, env, dir, "commit", "-q", "--allow-empty", "-m", "base") + } + runFixtureGit(t, env, f.main, "worktree", "add", "-q", f.worktree) + runFixtureGit(t, env, f.main, "worktree", "add", "-q", f.relWorktree) + // A relative gitfile, as `git worktree add --relative-paths` (or a moved + // checkout) writes it. + if err := os.WriteFile(filepath.Join(f.relWorktree, ".git"), []byte("gitdir: ../main/.git/worktrees/wt-rel\n"), 0o600); err != nil { + t.Fatal(err) + } + // A gitfile naming its git directory through a symlink: Git prints the + // resolved path. + runFixtureGit(t, env, f.main, "worktree", "add", "-q", f.symWorktree) + if err := os.Symlink(filepath.Join(f.main, ".git"), filepath.Join(root, "gitlink")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(f.symWorktree, ".git"), []byte("gitdir: "+filepath.Join(root, "gitlink", "worktrees", "wt-sym")+"\n"), 0o600); err != nil { + t.Fatal(err) + } + runFixtureGit(t, env, f.main, "-c", "protocol.file.allow=always", "submodule", "add", "-q", subSrc, "sub") + runFixtureGit(t, env, f.nested, "init", "-q") + for _, dir := range []string{filepath.Join(f.worktree, "w"), filepath.Join(f.submodule, "s"), filepath.Join(f.nested, "deep")} { + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(f.main, filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + return f +} + +// TestDiscoverGitFromMatchesRevParse is the equivalence contract: wherever +// in-process discovery answers, its answer is byte-for-byte what `git +// rev-parse --git-dir --git-common-dir --show-toplevel` prints there. +func TestDiscoverGitFromMatchesRevParse(t *testing.T) { + f := newDiscoveryFixture(t) + for _, tc := range []struct{ name, dir string }{ + {"repo top", f.main}, + {"repo subdir", filepath.Join(f.main, "a", "b")}, + {"repo first-level subdir", filepath.Join(f.main, "a")}, + {"linked worktree top", f.worktree}, + {"linked worktree subdir", filepath.Join(f.worktree, "w")}, + {"linked worktree with relative gitfile", f.relWorktree}, + {"linked worktree with gitfile through a symlink", f.symWorktree}, + {"submodule top", f.submodule}, + {"submodule subdir", filepath.Join(f.submodule, "s")}, + {"nested repo top", f.nested}, + {"nested repo subdir", filepath.Join(f.nested, "deep")}, + {"symlinked path to repo top", filepath.Join(f.root, "link")}, + {"symlinked path to repo subdir", filepath.Join(f.root, "link", "a", "b")}, + {"no repository", filepath.Join(f.root, "norepo", "x")}, + } { + t.Run(tc.name, func(t *testing.T) { + got, ok := discoverGitFrom(tc.dir) + if !ok { + if tc.name == "no repository" { + // An ancestor of the temp dir may be unusual (a bare + // repository, a foreign owner); declining is always safe. + t.Skip("in-process discovery declined above the fixture root") + } + t.Fatalf("discoverGitFrom(%s) declined; want an in-process answer", tc.dir) + } + if want := gitRevParse(t, f.env, tc.dir); got != want { + t.Fatalf("discoverGitFrom(%s) = %+v, git rev-parse = %+v", tc.dir, got, want) + } + }) + } +} + +// TestDiscoverGitFromDeclinesUnmodeledLayouts covers the layouts in-process +// discovery must hand to git rather than guess. +func TestDiscoverGitFromDeclinesUnmodeledLayouts(t *testing.T) { + f := newDiscoveryFixture(t) + repoWithConfig := func(t *testing.T, args ...string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "r") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", dir) + runFixtureGit(t, f.env, dir, append([]string{"config"}, args...)...) + return dir + } + invalidGitfile := filepath.Join(t.TempDir(), "badfile") + if err := os.MkdirAll(invalidGitfile, 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(invalidGitfile, ".git"), []byte("not a gitfile\n"), 0o600); err != nil { + t.Fatal(err) + } + symlinkedDotGit := filepath.Join(t.TempDir(), "symlinked") + if err := os.MkdirAll(symlinkedDotGit, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(f.main, ".git"), filepath.Join(symlinkedDotGit, ".git")); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ name, dir string }{ + {"core.worktree", repoWithConfig(t, "core.worktree", t.TempDir())}, + {"core.bare", repoWithConfig(t, "core.bare", "true")}, + {"extensions.worktreeConfig", repoWithConfig(t, "extensions.worktreeConfig", "true")}, + {"include directive", repoWithConfig(t, "include.path", "/dev/null")}, + {"inside the git directory", filepath.Join(f.main, ".git", "refs")}, + {"invalid gitfile", invalidGitfile}, + {"symlinked .git", symlinkedDotGit}, + } { + t.Run(tc.name, func(t *testing.T) { + if got, ok := discoverGitFrom(tc.dir); ok { + t.Fatalf("discoverGitFrom(%s) = %+v; want it to defer to git", tc.dir, got) + } + }) + } +} + +// TestDiscoverGitInProcessDefersToGitEnvOverrides: GIT_DIR and friends +// redefine discovery, so their presence (even empty) sends it to git. +func TestDiscoverGitInProcessDefersToGitEnvOverrides(t *testing.T) { + f := newDiscoveryFixture(t) + t.Chdir(f.main) + for _, name := range discoveryEnvOverrides { + t.Run(name, func(t *testing.T) { + t.Setenv(name, "") + if got, ok := discoverGitInProcess(); ok { + t.Fatalf("with %s set, discoverGitInProcess() = %+v; want it to defer to git", name, got) + } + }) + } +} + +// TestGitContextInProcessMatchesGitSubprocess checks the cached context end +// to end: what bd's startup now derives without git equals what it derived +// from the git subprocess, for each discovery layout. +func TestGitContextInProcessMatchesGitSubprocess(t *testing.T) { + f := newDiscoveryFixture(t) + for _, k := range discoveryEnvOverrides { + if _, set := os.LookupEnv(k); set { + t.Setenv(k, "") + if err := os.Unsetenv(k); err != nil { + t.Fatal(err) + } + } + } + for _, dir := range []string{ + f.main, filepath.Join(f.main, "a", "b"), f.worktree, filepath.Join(f.worktree, "w"), + f.submodule, filepath.Join(f.nested, "deep"), + } { + t.Run(strings.TrimPrefix(dir, f.root), func(t *testing.T) { + t.Chdir(dir) + raw, ok := discoverGitInProcess() + if !ok { + t.Fatalf("discoverGitInProcess declined in %s", dir) + } + got := gitContextFromRevParse("", raw) + want := loadGitContext("", f.env) + if got.err != nil || want.err != nil { + t.Fatalf("errors: in-process %v, git %v", got.err, want.err) + } + if got.gitDirRaw != want.gitDirRaw || got.commonDir != want.commonDir || + got.repoRoot != want.repoRoot || got.isWorktree != want.isWorktree { + t.Fatalf("in-process %+v, git %+v", got, want) + } + }) + } +} diff --git a/internal/git/discover_other.go b/internal/git/discover_other.go new file mode 100644 index 000000000..4ba6e1083 --- /dev/null +++ b/internal/git/discover_other.go @@ -0,0 +1,10 @@ +//go:build !linux + +package git + +// discoverGitInProcess is linux-only: elsewhere path case-folding (darwin, +// windows) and ownership semantics (windows) make an exact in-process answer +// harder to prove, so discovery always runs git. +func discoverGitInProcess() (revParseResult, bool) { + return revParseResult{}, false +} diff --git a/internal/git/gitdir.go b/internal/git/gitdir.go index 31019d7b4..bb53eb63a 100644 --- a/internal/git/gitdir.go +++ b/internal/git/gitdir.go @@ -88,10 +88,35 @@ func canonicalPinPath(p string) string { // initGitContext populates the gitContext with a single git call. // This is called once per process via sync.Once. +// +// The process-wide context is the one every bd invocation resolves at startup +// (beads directory discovery), so it first tries discoverGitInProcess, which +// answers the common layouts — an ordinary repository, a linked worktree, a +// submodule, no repository at all — from the filesystem with the exact output +// `git rev-parse --git-dir --git-common-dir --show-toplevel` would print, and +// declines (falling back to that subprocess) for anything it cannot prove +// equivalent: GIT_DIR-style overrides, repository config that moves the work +// tree, ownership that would engage safe.directory, and so on. func initGitContext() { + if raw, ok := discoverGitInProcess(); ok { + gitCtx = gitContextFromRevParse("", raw) + return + } gitCtx = loadGitContext("", nil) } +// revParseResult is the answer to `git rev-parse --git-dir --git-common-dir +// --show-toplevel`, in Git's own spelling, or notRepo when Git would report +// that the directory is not inside a repository. +type revParseResult struct { + gitDir, commonDir, topLevel string + notRepo bool +} + +// errNotGitRepository stands in for git's exit status when in-process +// discovery establishes that there is no repository to find. +var errNotGitRepository = errors.New("no .git found in the working directory or any parent") + func loadGitContext(workDir string, env []string) gitContext { var ctx gitContext // Get all three values with a single git call @@ -116,10 +141,25 @@ func loadGitContext(workDir string, env []string) gitContext { ctx.err = fmt.Errorf("unexpected git rev-parse output: got %d lines, expected 3", len(lines)) return ctx } + return gitContextFromRevParse(workDir, revParseResult{ + gitDir: strings.TrimSpace(lines[0]), + commonDir: strings.TrimSpace(lines[1]), + topLevel: strings.TrimSpace(lines[2]), + }) +} - ctx.gitDirRaw = strings.TrimSpace(lines[0]) - commonDirRaw := strings.TrimSpace(lines[1]) - repoRootRaw := strings.TrimSpace(lines[2]) +// gitContextFromRevParse derives the cached context from rev-parse output, +// whichever of git or discoverGitInProcess produced it, so both paths share +// one normalization. +func gitContextFromRevParse(workDir string, raw revParseResult) gitContext { + var ctx gitContext + if raw.notRepo { + ctx.err = fmt.Errorf("not a git repository: %w", errNotGitRepository) + return ctx + } + ctx.gitDirRaw = raw.gitDir + commonDirRaw := raw.commonDir + repoRootRaw := raw.topLevel // Convert commonDir to absolute for reliable comparison absCommon, err := absoluteGitPath(workDir, commonDirRaw) From 615d6508bc40c7d42526aa0543062b3427fdaf56 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 18:48:39 +0000 Subject: [PATCH 002/151] perf(cli): resolve the git user.name actor fallback only when needed PersistentPreRunE assigned `actor = getActorWithGit()` for every command, which, with no --actor / BEADS_ACTOR / BD_ACTOR / config actor, runs `git config user.name`: the second git subprocess of every bd call, paid by read-only commands that never consult the actor. The pre-run now calls deferActorGitFallback, which marks an empty actor as pending; currentActor (and getActorWithGit, getActor) resolve it on first use with the unchanged priority order and store it in the global and the CommandContext copy, as the eager assignment did. Command code reads the actor through currentActor() instead of the raw global (a mechanical rewrite of every read site the compiler reported; writes, the flag binding and the pre-run config binding keep the raw variable). setActor cancels a pending fallback. The telemetry span's bd.actor attribute is still set, but only for a recording span, so a noop span does not force the lookup. With the in-process repository discovery from the previous commit, a read-only `bd list` / `bd show` now starts no git process at startup, in the main checkout and in a linked worktree; a write asks git for user.name once (bd create previously asked once, now still once) and records the same value. Measured on 50 sequential `bd list` calls: 100 -> 0 git execs (embedded and proxied-server workspaces, main checkout and worktree). TestEmbeddedReadOnlyCommandsSpawnNoGit counts git execs through a PATH shim; TestDeferredActorGitFallback pins the deferral semantics. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/scripts/embedded-cmd-test-shards.txt | 1 + CHANGELOG.md | 12 ++ cmd/bd/BUILD.bazel | 1 + cmd/bd/actor_test.go | 54 +++++++ cmd/bd/ado.go | 6 +- cmd/bd/assign.go | 6 +- cmd/bd/audit.go | 8 +- cmd/bd/close.go | 10 +- cmd/bd/close_direct.go | 4 +- cmd/bd/close_if_revision.go | 12 +- cmd/bd/close_proxied_server.go | 10 +- cmd/bd/comment.go | 2 +- cmd/bd/comments.go | 2 +- cmd/bd/context.go | 11 +- cmd/bd/cook.go | 6 +- cmd/bd/create.go | 4 +- cmd/bd/create_form.go | 2 +- cmd/bd/defer.go | 2 +- cmd/bd/defer_proxied_server.go | 4 +- cmd/bd/delete.go | 4 +- cmd/bd/delete_proxied_server.go | 2 +- cmd/bd/dep.go | 10 +- cmd/bd/dep_proxied_server.go | 4 +- cmd/bd/duplicates.go | 2 +- cmd/bd/edit.go | 6 +- cmd/bd/gate.go | 12 +- cmd/bd/gate_discover.go | 2 +- cmd/bd/gate_proxied_server.go | 16 +-- cmd/bd/gc_proxied_server.go | 2 +- cmd/bd/github.go | 2 +- cmd/bd/gitlab.go | 2 +- cmd/bd/graph_apply.go | 2 +- cmd/bd/heartbeat.go | 6 +- cmd/bd/heartbeat_proxied_server.go | 2 +- cmd/bd/human.go | 6 +- cmd/bd/human_proxied_server.go | 8 +- cmd/bd/jira.go | 2 +- cmd/bd/label.go | 8 +- cmd/bd/label_proxied_server.go | 6 +- cmd/bd/linear.go | 6 +- cmd/bd/link.go | 4 +- cmd/bd/link_proxied_server.go | 2 +- cmd/bd/main.go | 78 +++++++++- cmd/bd/markdown.go | 4 +- cmd/bd/merge_slot.go | 8 +- cmd/bd/migrate_issues.go | 2 +- cmd/bd/mol_bond.go | 12 +- cmd/bd/mol_burn.go | 4 +- cmd/bd/mol_current.go | 2 +- cmd/bd/mol_progress.go | 2 +- cmd/bd/mol_proxied_server.go | 20 +-- cmd/bd/mol_squash.go | 2 +- cmd/bd/mutate_proxied_server.go | 16 +-- cmd/bd/note.go | 4 +- cmd/bd/notion.go | 2 +- cmd/bd/pour.go | 4 +- cmd/bd/priority.go | 4 +- cmd/bd/promote.go | 4 +- cmd/bd/promote_proxied_server.go | 4 +- cmd/bd/purge.go | 2 +- cmd/bd/quick.go | 2 +- cmd/bd/quick_proxied_server.go | 2 +- cmd/bd/ready.go | 2 +- cmd/bd/ready_input.go | 2 +- cmd/bd/reclaim.go | 4 +- cmd/bd/relate.go | 8 +- cmd/bd/relate_proxied_server.go | 6 +- cmd/bd/rename_prefix.go | 4 +- cmd/bd/reopen.go | 4 +- cmd/bd/reopen_proxied_server.go | 4 +- cmd/bd/ship.go | 2 +- cmd/bd/startup_git_exec_embedded_test.go | 141 +++++++++++++++++++ cmd/bd/state.go | 4 +- cmd/bd/status.go | 2 +- cmd/bd/swarm.go | 16 +-- cmd/bd/sync_push_pull.go | 26 ++-- cmd/bd/tag.go | 4 +- cmd/bd/unclaim.go | 6 +- cmd/bd/unclaim_proxied_server.go | 8 +- cmd/bd/undefer.go | 2 +- cmd/bd/update.go | 12 +- cmd/bd/update_proxied_server.go | 4 +- cmd/bd/wisp.go | 2 +- cmd/bd/wisp_proxied_server.go | 6 +- 84 files changed, 502 insertions(+), 218 deletions(-) create mode 100644 cmd/bd/startup_git_exec_embedded_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index b1c7b0e8c..7a1c26452 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -306,6 +306,7 @@ 50 17 TestEmbeddedCompact 50 17 TestEmbeddedGateConcurrent 50 17 TestEmbeddedListComments +50 17 TestEmbeddedReadOnlyCommandsSpawnNoGit 50 18 TestEmbeddedContext 50 18 TestEmbeddedContributorCreate diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d65701d..98882f789 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Every `bd` invocation used to start two git subprocesses before doing any + work: `git rev-parse` to locate the repository and `git config user.name` + to resolve the actor. On Linux the repository is now located in-process + (ordinary repositories, linked worktrees, submodules, no repository), + producing exactly what `git rev-parse` printed and still running git for + anything it does not model (`GIT_DIR` and related overrides, + `core.worktree`/`core.bare`, repository extensions, includes, foreign + ownership subject to `safe.directory`). The `git config user.name` actor + fallback is resolved only when a command actually needs an actor, so + read-only commands such as `bd list` and `bd show` start no git process + for it. The actor's value and priority order are unchanged. + - `bd create --graph` now stores a plan whose path from a node's parent to the node runs through a `waits-for` edge; the graph-only preflight that walked every ready-work edge used to refuse it. The plan now goes through diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 8f5ea468f..5964f512a 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1147,6 +1147,7 @@ go_test( "sql_test.go", "stale_embedded_test.go", "stale_test.go", + "startup_git_exec_embedded_test.go", "state_embedded_test.go", "state_proxied_integration_test.go", "state_test.go", diff --git a/cmd/bd/actor_test.go b/cmd/bd/actor_test.go index d7efa7fc3..5ef83b505 100644 --- a/cmd/bd/actor_test.go +++ b/cmd/bd/actor_test.go @@ -246,3 +246,57 @@ func TestGetActorWithGit_PriorityOrder(t *testing.T) { t.Errorf("Expected BD_ACTOR to be used as fallback, got %q", result) } } + +// TestDeferredActorGitFallback pins the lazy actor: PersistentPreRunE defers +// the git config user.name fallback, and the first reader resolves it to +// exactly what the former eager getActorWithGit assignment produced. +func TestDeferredActorGitFallback(t *testing.T) { + origActor, origPending := actor, actorGitFallbackPending + var origCtxActor string + if cmdCtx != nil { + origCtxActor = cmdCtx.Actor + } + t.Cleanup(func() { + actor, actorGitFallbackPending = origActor, origPending + if cmdCtx != nil { + cmdCtx.Actor = origCtxActor + } + }) + t.Setenv("BEADS_ACTOR", "") + t.Setenv("BD_ACTOR", "") + + actor = "explicit" + deferActorGitFallback() + if actorGitFallbackPending { + t.Fatal("a configured actor must not defer a git lookup") + } + if got := currentActor(); got != "explicit" { + t.Fatalf("currentActor() = %q, want %q", got, "explicit") + } + + actor = "" + want := resolveActorWithGit() + deferActorGitFallback() + if !actorGitFallbackPending || actor != "" { + t.Fatalf("deferral resolved eagerly: pending=%v actor=%q", actorGitFallbackPending, actor) + } + if got := currentActor(); got != want { + t.Fatalf("currentActor() = %q, want eager resolution %q", got, want) + } + if actorGitFallbackPending || actor != want { + t.Fatalf("after first use: pending=%v actor=%q, want resolved %q", actorGitFallbackPending, actor, want) + } + + actor = "" + deferActorGitFallback() + if got := getActorWithGit(); got != want || actorGitFallbackPending || actor != want { + t.Fatalf("getActorWithGit() = %q (pending=%v actor=%q), want %q resolved into the global", got, actorGitFallbackPending, actor, want) + } + + actor = "" + deferActorGitFallback() + setActor("assigned") + if got := currentActor(); got != "assigned" { + t.Fatalf("setActor must cancel a deferred fallback: currentActor() = %q", got) + } +} diff --git a/cmd/bd/ado.go b/cmd/bd/ado.go index 08206992b..5e54257bc 100644 --- a/cmd/bd/ado.go +++ b/cmd/bd/ado.go @@ -540,7 +540,7 @@ func runADOSync(cmd *cobra.Command, _ []string) error { } // Create the sync engine - engine := tracker.NewEngine(at, store, actor) + engine := tracker.NewEngine(at, store, currentActor()) var warnings []string if !jsonOutput { engine.OnMessage = func(msg string) { _, _ = fmt.Fprintln(out, " "+msg) } @@ -641,7 +641,7 @@ func runADOSync(cmd *cobra.Command, _ []string) error { continue } reason := fmt.Sprintf("ADO work item %s deleted", idStr) - if cerr := store.CloseIssue(ctx, localID, reason, actor, ""); cerr != nil { + if cerr := store.CloseIssue(ctx, localID, reason, currentActor(), ""); cerr != nil { msg := fmt.Sprintf("Failed to close %s for deleted ADO #%s: %v", localID, idStr, cerr) warnings = append(warnings, msg) if !jsonOutput { @@ -915,7 +915,7 @@ func buildADOPullHooks(ctx context.Context, at *ado.Tracker, bootstrapMatch, noC "external_ref": ref, "source_system": "ado:" + extIssue.ID, } - if err := store.UpdateIssue(ctx, result.BeadsID, updates, actor); err == nil { + if err := store.UpdateIssue(ctx, result.BeadsID, updates, currentActor()); err == nil { *matchCount++ if warn != nil { warn(fmt.Sprintf("Bootstrap matched ADO #%s → %s (%s)", extIssue.ID, result.BeadsID, result.MatchType)) diff --git a/cmd/bd/assign.go b/cmd/bd/assign.go index ba3a72e97..83b145f33 100644 --- a/cmd/bd/assign.go +++ b/cmd/bd/assign.go @@ -85,7 +85,7 @@ Examples: // lost race reports precondition_failed from the guarded write below // instead of this policy refusal — see ifRevisionAlreadyStale's doc. if !ifRevisionAlreadyStale(result.Issue, ifRevision) { - if err := validateIssueReassignable(id, result.Issue, actor, assignee, + if err := validateIssueReassignable(id, result.Issue, currentActor(), assignee, storeClaimPoolAliases(ctx, issueStore), force); err != nil { return HandleErrorRespectJSON("%s", err) } @@ -104,7 +104,7 @@ Examples: return HandleErrorRespectJSON("%v", err) } mutationResult, err := runCommandUpdateMutation(opsCtx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: result.ResolvedID, patch: issueops.IssuePatch{ Assignee: issueops.Field[string]{Set: true, Value: assignee}, @@ -121,7 +121,7 @@ Examples: return HandleErrorRespectJSON("updating %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "assign", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/audit.go b/cmd/bd/audit.go index 127e3f236..fd33d0536 100644 --- a/cmd/bd/audit.go +++ b/cmd/bd/audit.go @@ -77,8 +77,8 @@ var auditRecordCmd = &cobra.Command{ if err := json.Unmarshal(b, &e); err != nil { return HandleError("invalid JSON on stdin: %v", err) } - if actor != "" { - e.Actor = actor + if currentActor() != "" { + e.Actor = currentActor() } } else { if auditRecordKind == "" { @@ -86,7 +86,7 @@ var auditRecordCmd = &cobra.Command{ } e = audit.Entry{ Kind: auditRecordKind, - Actor: actor, + Actor: currentActor(), IssueID: auditRecordIssueID, Model: auditRecordModel, Prompt: auditRecordPrompt, @@ -137,7 +137,7 @@ var auditLabelCmd = &cobra.Command{ } e := audit.Entry{ Kind: "label", - Actor: actor, + Actor: currentActor(), ParentID: parentID, Label: auditLabelValue, Reason: auditLabelReason, diff --git a/cmd/bd/close.go b/cmd/bd/close.go index b99abe219..5a28910a2 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -227,7 +227,7 @@ the flags appear in the command line.`, // suppressed real-close side effects (no audit, no closed→closed on the // step). Register the store when it actually closed the root so the // pending-commit sweep persists it — closedCount==0 would not commit. - if molID := autoCloseCompletedMolecule(ctx, activeStore, id, actor, session); molID != "" { + if molID := autoCloseCompletedMolecule(ctx, activeStore, id, currentActor(), session); molID != "" { mutatedStores[activeStore] = append(mutatedStores[activeStore], molID) } } else { @@ -238,13 +238,13 @@ the flags appear in the command line.`, if issue != nil { oldStatus = string(issue.Status) } - audit.LogFieldChange(id, "status", oldStatus, "closed", actor, reason) + audit.LogFieldChange(id, "status", oldStatus, "closed", currentActor(), reason) closedCount++ // Auto-close parent molecule if all steps are now complete. // Runs against the same store the step was closed in. - autoCloseCompletedMolecule(ctx, activeStore, id, actor, session) + autoCloseCompletedMolecule(ctx, activeStore, id, currentActor(), session) } // First id this command settled as closed — a real close or an @@ -313,7 +313,7 @@ the flags appear in the command line.`, if continueFlag && len(resolvedIDs) == 1 && closedForCommand { autoClaim := !noAuto - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(postCloseStore), resolvedIDs[0], autoClaim, actor) + result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(postCloseStore), resolvedIDs[0], autoClaim, currentActor()) if err != nil { fmt.Fprintf(os.Stderr, "Warning: could not advance to next step: %v\n", err) } else if result != nil { @@ -400,7 +400,7 @@ the flags appear in the command line.`, if s == nil { continue } - if err := commitPendingIfEmbedded(ctx, s, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, s, currentActor(), doltAutoCommitParams{ Command: "close", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/close_direct.go b/cmd/bd/close_direct.go index d7ac30cb1..47890945e 100644 --- a/cmd/bd/close_direct.go +++ b/cmd/bd/close_direct.go @@ -81,7 +81,7 @@ func closeDirectCheckOne(id string, issue *types.Issue, st storage.DoltStorage, // guard remains the authority on whether the close is a no-op, so a // concurrent close still converges. Mirrored in closeProxiedCheckOne. if issue == nil || issue.Status != types.StatusClosed { - if err := validateIssueClosable(id, issue, actor, force); err != nil { + if err := validateIssueClosable(id, issue, currentActor(), force); err != nil { return err.Error() } } @@ -135,7 +135,7 @@ func closeDirectBatches(items []closeDirectItem) []closeDirectBatch { // hands out ONE claim however many stores the ids spanned. func closeDirectRequest(batch closeDirectBatch, session string, force bool, claimStore storage.DoltStorage, claimNext *issueops.ReadyRequest) issueops.CloseBatchRequest { request := issueops.CloseBatchRequest{ - Actor: actor, + Actor: currentActor(), Items: make([]issueops.BatchCloseItem, 0, len(batch.items)), Session: session, Force: force, diff --git a/cmd/bd/close_if_revision.go b/cmd/bd/close_if_revision.go index ac5bfda1a..b544714e5 100644 --- a/cmd/bd/close_if_revision.go +++ b/cmd/bd/close_if_revision.go @@ -50,7 +50,7 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool preCloseStatus := string(result.Issue.Status) closeResult, closeErr := ops.Close(opsCtx, issueops.CloseRequest{ - Actor: actor, + Actor: currentActor(), IssueID: result.ResolvedID, Reason: reason, Session: session, @@ -79,11 +79,11 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool // entirely (mc-zndi7.75) and would otherwise leave a molecule's root // stranded open forever. mutatedIDs := []string{result.ResolvedID} - if molID := autoCloseCompletedMolecule(ctx, result.Store, result.ResolvedID, actor, session); molID != "" { + if molID := autoCloseCompletedMolecule(ctx, result.Store, result.ResolvedID, currentActor(), session); molID != "" { mutatedIDs = append(mutatedIDs, molID) } - if err := commitPendingIfEmbedded(ctx, result.Store, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, result.Store, currentActor(), doltAutoCommitParams{ Command: "close", IssueIDs: mutatedIDs, }); err != nil { @@ -116,7 +116,7 @@ func runCloseProxiedIfRevision(ctx context.Context, id, reason string, force boo } closeResult, closeErr := ops.Close(ctx, issueops.CloseRequest{ - Actor: actor, + Actor: currentActor(), IssueID: id, Reason: reason, Session: session, @@ -165,7 +165,7 @@ func runCloseIfRevisionProxiedPostClose(ctx context.Context, id, session string) } autoClosedMol, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*types.Issue, string, error) { var warnings []string - mol := autoCloseProxiedCompletedMolecule(ctx, uw, id, actor, session, &warnings) + mol := autoCloseProxiedCompletedMolecule(ctx, uw, id, currentActor(), session, &warnings) for _, w := range warnings { fmt.Fprintf(os.Stderr, "Warning: %s\n", w) } @@ -209,7 +209,7 @@ func reportClosedIfRevisionResult(id, reason, preCloseStatus string, closeResult closedIssue.Dependencies = nil } if closeResult.Changed { - audit.LogFieldChange(id, "status", preCloseStatus, "closed", actor, reason) + audit.LogFieldChange(id, "status", preCloseStatus, "closed", currentActor(), reason) } if jsonOutput { if closedIssue != nil { diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index 94a614e3a..e7a791d4f 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -140,7 +140,7 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin return HandleErrorRespectJSON("%v", cerr) } result, err = closer.CloseBatch(ctx, issueops.CloseBatchRequest{ - Actor: actor, + Actor: currentActor(), Items: pre.items, Session: in.session, Force: in.force, @@ -166,7 +166,7 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin for i, o := range outcomes { if o.closed { - audit.LogFieldChange(o.id, "status", o.auditOld, "closed", actor, o.auditReason) + audit.LogFieldChange(o.id, "status", o.auditOld, "closed", currentActor(), o.auditReason) } if !in.jsonOut { fmt.Printf("%s Closed %s: %s\n", ui.RenderPass("✓"), formatFeedbackID(o.after.ID, o.after.Title), closeReasons[i]) @@ -300,7 +300,7 @@ func closeProxiedCheckOne(ctx context.Context, uw uow.UnitOfWork, id string, in // always been. Both close paths must agree here — diverging is the defect // class #5217 closed. if current.Status != types.StatusClosed { - if err := validateIssueClosable(id, current, actor, in.force); err != nil { + if err := validateIssueClosable(id, current, currentActor(), in.force); err != nil { return err.Error(), nil } } @@ -483,7 +483,7 @@ func closeProxiedRunPostClose(ctx context.Context, args []string, in closeProxie var wrote []string for _, o := range outcomes { - mol := autoCloseProxiedCompletedMolecule(ctx, uw, o.id, actor, in.session, &out.warnings) + mol := autoCloseProxiedCompletedMolecule(ctx, uw, o.id, currentActor(), in.session, &out.warnings) if mol != nil { out.autoClosedMol = mol wrote = append(wrote, "auto-close "+mol.ID) @@ -529,7 +529,7 @@ func closeProxiedSuggestNext(ctx context.Context, uw uow.UnitOfWork, closedID st } func closeProxiedContinue(ctx context.Context, uw uow.UnitOfWork, closedID string, autoClaim bool) (*ContinueResult, string) { - result, err := AdvanceToNextStep(ctx, newUOWMolWriter(uw), closedID, autoClaim, actor) + result, err := AdvanceToNextStep(ctx, newUOWMolWriter(uw), closedID, autoClaim, currentActor()) if err != nil { return nil, fmt.Sprintf("could not advance to next step: %v", err) } diff --git a/cmd/bd/comment.go b/cmd/bd/comment.go index 15f577e7c..76cf3342a 100644 --- a/cmd/bd/comment.go +++ b/cmd/bd/comment.go @@ -183,7 +183,7 @@ To list comments on an issue, use the plural form: bd comments `, if err != nil { return HandleErrorRespectJSON("adding comment: %v", err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "comment", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/comments.go b/cmd/bd/comments.go index c789ef498..256e6151f 100644 --- a/cmd/bd/comments.go +++ b/cmd/bd/comments.go @@ -258,7 +258,7 @@ Examples: if err != nil { return HandleErrorRespectJSON("adding comment: %v", err) } - if err := commitPendingIfEmbedded(ctx, result.Store, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, result.Store, currentActor(), doltAutoCommitParams{ Command: "comments add", IssueIDs: []string{issueID}, }); err != nil { diff --git a/cmd/bd/context.go b/cmd/bd/context.go index 6b4807e47..bd772896b 100644 --- a/cmd/bd/context.go +++ b/cmd/bd/context.go @@ -112,17 +112,26 @@ func setStore(s storage.DoltStorage) { // getActor returns the current actor name for audit trail. func getActor() string { if shouldUseGlobals() { - return actor + return currentActor() + } + // cmdCtx.Actor is a copy of the global taken by syncCommandContext or + // setActor. While the two agree, the global is authoritative and may + // still carry a deferred git fallback (see deferActorGitFallback). + if cmdCtx.Actor == actor { + return currentActor() } return cmdCtx.Actor } // setActor updates the actor name in the CommandContext. func setActor(a string) { + actorMu.Lock() + defer actorMu.Unlock() if cmdCtx != nil { cmdCtx.Actor = a } actor = a + actorGitFallbackPending = false } // isJSONOutput returns true if JSON output mode is enabled. diff --git a/cmd/bd/cook.go b/cmd/bd/cook.go index d76422c97..6e94309b0 100644 --- a/cmd/bd/cook.go +++ b/cmd/bd/cook.go @@ -1001,20 +1001,20 @@ func cookFormula(ctx context.Context, s storage.DoltStorage, f *formula.Formula, } // Create all issues - if err := tx.CreateIssues(ctx, issues, actor); err != nil { + if err := tx.CreateIssues(ctx, issues, currentActor()); err != nil { return fmt.Errorf("failed to create issues: %w", err) } // Add labels for _, l := range labels { - if err := tx.AddLabel(ctx, l.issueID, l.label, actor); err != nil { + if err := tx.AddLabel(ctx, l.issueID, l.label, currentActor()); err != nil { return fmt.Errorf("failed to add label %s to %s: %w", l.label, l.issueID, err) } } // Add dependencies for _, dep := range deps { - if err := tx.AddDependency(ctx, dep, actor); err != nil { + if err := tx.AddDependency(ctx, dep, currentActor()); err != nil { return fmt.Errorf("failed to create dependency: %w", err) } } diff --git a/cmd/bd/create.go b/cmd/bd/create.go index e65c9ab82..38692d287 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -568,7 +568,7 @@ var createCmd = &cobra.Command{ // dry-run preview needs it too; asking the facade to inherit as well // would append the parent's labels a second time. result, err := ops.Create(opsCtx, issueops.CreateRequest{ - Actor: actor, + Actor: currentActor(), Issue: issue, ParentID: parentID, Dependencies: createDependencyRequests(depSpecs), @@ -610,7 +610,7 @@ var createCmd = &cobra.Command{ } if repoPath != "." && targetStore != nil { - if err := commitPendingIfEmbedded(ctx, targetStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, targetStore, currentActor(), doltAutoCommitParams{ Command: "create", IssueIDs: []string{created.ID}, }); err != nil { diff --git a/cmd/bd/create_form.go b/cmd/bd/create_form.go index d52f88dce..42d455796 100644 --- a/cmd/bd/create_form.go +++ b/cmd/bd/create_form.go @@ -371,7 +371,7 @@ func runCreateForm(cmd *cobra.Command) error { fv := parseCreateFormInput(raw) fv.ParentID = parentID - issue, err := CreateIssueFromFormValues(rootCtx, store, fv, actor) + issue, err := CreateIssueFromFormValues(rootCtx, store, fv, currentActor()) if err != nil { return HandleError("%v", err) } diff --git a/cmd/bd/defer.go b/cmd/bd/defer.go index 4158253bb..0a22b18be 100644 --- a/cmd/bd/defer.go +++ b/cmd/bd/defer.go @@ -123,7 +123,7 @@ Examples: updates["notes"] = notes + reason } - if err := store.UpdateIssue(ctx, fullID, updates, actor); err != nil { + if err := store.UpdateIssue(ctx, fullID, updates, currentActor()); err != nil { fmt.Fprintf(os.Stderr, "Error deferring %s: %v\n", fullID, err) continue } diff --git a/cmd/bd/defer_proxied_server.go b/cmd/bd/defer_proxied_server.go index 280df6ace..0e1a47847 100644 --- a/cmd/bd/defer_proxied_server.go +++ b/cmd/bd/defer_proxied_server.go @@ -27,9 +27,9 @@ type deferProxiedResult struct { func proxiedUpdateByID(ctx context.Context, uw uow.UnitOfWork, id string, isWisp bool, updates map[string]any) error { if isWisp { - return uw.IssueUseCase().UpdateWisp(ctx, id, updates, actor) + return uw.IssueUseCase().UpdateWisp(ctx, id, updates, currentActor()) } - return uw.IssueUseCase().UpdateIssue(ctx, id, updates, actor) + return uw.IssueUseCase().UpdateIssue(ctx, id, updates, currentActor()) } func proxiedGetByID(ctx context.Context, uw uow.UnitOfWork, id string, isWisp bool) *types.Issue { diff --git a/cmd/bd/delete.go b/cmd/bd/delete.go index d48de774a..c767f3958 100644 --- a/cmd/bd/delete.go +++ b/cmd/bd/delete.go @@ -163,7 +163,7 @@ Force: Delete and orphan dependents // rewrite runs INSIDE the transaction that deletes, because it is the // role's. request := issueops.DeleteRequest{ - Actor: actor, + Actor: currentActor(), IDs: []string{issueID}, Force: force, DryRun: dryRun || !force, @@ -378,7 +378,7 @@ func deleteBatch(_ *cobra.Command, issueIDs []string, force bool, dryRun bool, c // --force is the confirmation as well as the orphan mode, so an unconfirmed // run asks the role what it WOULD do; see the single-id path. request := issueops.DeleteRequest{ - Actor: actor, + Actor: currentActor(), IDs: resolvedIDs, Cascade: cascade, Force: force, diff --git a/cmd/bd/delete_proxied_server.go b/cmd/bd/delete_proxied_server.go index 9cd6b0491..3fff3137d 100644 --- a/cmd/bd/delete_proxied_server.go +++ b/cmd/bd/delete_proxied_server.go @@ -102,7 +102,7 @@ func runDeleteProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri // --force is the confirmation as well as the orphan mode, exactly as on the // direct route, so an unconfirmed run asks the role what it WOULD do. request := issueops.DeleteRequest{ - Actor: actor, + Actor: currentActor(), IDs: in.ids, Cascade: in.cascade, Force: in.force, diff --git a/cmd/bd/dep.go b/cmd/bd/dep.go index ef502d810..e77e1e1c8 100644 --- a/cmd/bd/dep.go +++ b/cmd/bd/dep.go @@ -37,7 +37,7 @@ func addDependencyEdgesDirect(ctx context.Context, st storage.DoltStorage, edges return err } _, err = editor.AddDependencies(ctx, issueops.AddDependenciesRequest{ - Actor: actor, + Actor: currentActor(), Edges: edges, SkipPerEdgeCycleCheck: skipPerEdgeCycleCheck, }) @@ -330,7 +330,7 @@ Examples: warnIfCyclesExist(fromStore) } - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "dep add", IssueIDs: []string{fromID, toID}, }); err != nil { @@ -520,7 +520,7 @@ Examples: warnIfCyclesExist(fromStore) } - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "dep add", IssueIDs: []string{fromID, toID}, }); err != nil { @@ -1270,7 +1270,7 @@ var depRemoveCmd = &cobra.Command{ return HandleErrorRespectJSON("%v", err) } result, err := editor.RemoveDependency(opsCtx, issueops.RemoveDependencyRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fullFromID, DependsOnID: fullToID, }) @@ -1278,7 +1278,7 @@ var depRemoveCmd = &cobra.Command{ return HandleErrorRespectJSON("%v", err) } - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "dep remove", IssueIDs: []string{fullFromID, fullToID}, }); err != nil { diff --git a/cmd/bd/dep_proxied_server.go b/cmd/bd/dep_proxied_server.go index 28e50f79f..aee6b0306 100644 --- a/cmd/bd/dep_proxied_server.go +++ b/cmd/bd/dep_proxied_server.go @@ -76,7 +76,7 @@ func addDependencyEdgesProxied(ctx context.Context, edges []issueops.DependencyE return err } _, err = editor.AddDependencies(ctx, issueops.AddDependenciesRequest{ - Actor: actor, + Actor: currentActor(), Edges: edges, SkipPerEdgeCycleCheck: skipPerEdgeCycleCheck, }) @@ -370,7 +370,7 @@ func runDepRemoveProxiedServer(_ *cobra.Command, ctx context.Context, args []str return HandleErrorRespectJSON("%v", err) } result, err := editor.RemoveDependency(ctx, issueops.RemoveDependencyRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fromID, DependsOnID: toID, }) diff --git a/cmd/bd/duplicates.go b/cmd/bd/duplicates.go index 32c6417c0..f11e4ea35 100644 --- a/cmd/bd/duplicates.go +++ b/cmd/bd/duplicates.go @@ -424,7 +424,7 @@ func performMerge(targetID string, sourceIDs []string) map[string]interface{} { // Close the duplicate issue reason := fmt.Sprintf("Duplicate of %s", targetID) - if err := store.CloseIssue(ctx, sourceID, reason, actor, ""); err != nil { + if err := store.CloseIssue(ctx, sourceID, reason, currentActor(), ""); err != nil { errors = append(errors, fmt.Sprintf("failed to close %s: %v", sourceID, err)) continue } diff --git a/cmd/bd/edit.go b/cmd/bd/edit.go index 5fe2bed53..5e04128d0 100644 --- a/cmd/bd/edit.go +++ b/cmd/bd/edit.go @@ -150,7 +150,7 @@ Examples: fieldToEdit: newValue, } - err = issueStore.UpdateIssue(ctx, id, updates, actor) + err = issueStore.UpdateIssue(ctx, id, updates, currentActor()) if err != nil { if accessor, ok := storage.UnwrapStore(issueStore).(storage.RawDBAccessor); ok { if pingErr := accessor.DB().PingContext(ctx); pingErr != nil { @@ -158,14 +158,14 @@ Examples: _ = accessor.DB().PingContext(ctx) } } - err = issueStore.UpdateIssue(ctx, id, updates, actor) + err = issueStore.UpdateIssue(ctx, id, updates, currentActor()) } if err != nil { fmt.Fprintf(os.Stderr, "Your edits are preserved in: %s\n", tmpPath) return HandleErrorRespectJSON("updating issue: %v", err) } editSaved = true - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "edit", IssueIDs: []string{id}, }); err != nil { diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 3aa97289e..98985e062 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -275,7 +275,7 @@ This is used by 'bd done --phase-complete' to register for gate wake notificatio updates := map[string]interface{}{ "waiters": newWaiters, } - if err := store.UpdateIssue(ctx, gateID, updates, actor); err != nil { + if err := store.UpdateIssue(ctx, gateID, updates, currentActor()); err != nil { return HandleError("updating gate: %v", err) } @@ -355,7 +355,7 @@ Examples: } gate.Metadata = metadata - if err := store.CreateIssue(ctx, gate, actor); err != nil { + if err := store.CreateIssue(ctx, gate, currentActor()); err != nil { return HandleErrorRespectJSON("creating gate: %v", err) } @@ -364,7 +364,7 @@ Examples: DependsOnID: gate.ID, Type: types.DepBlocks, } - if err := store.AddDependency(ctx, dep, actor); err != nil { + if err := store.AddDependency(ctx, dep, currentActor()); err != nil { return HandleErrorRespectJSON("adding blocking dependency: %v", err) } @@ -573,7 +573,7 @@ Use --reason to provide context for why the gate was resolved.`, return HandleError("%s is not a gate issue (type=%s)", gateID, issue.IssueType) } - if err := store.CloseIssue(ctx, gateID, reason, actor, ""); err != nil { + if err := store.CloseIssue(ctx, gateID, reason, currentActor(), ""); err != nil { return HandleError("closing gate: %v", err) } @@ -679,7 +679,7 @@ Examples: return updateGateAwaitIDFunc(nil, gateID, runID) } recordSeen = func(gate *types.Issue) error { - if err := store.UpdateIssue(ctx, gate.ID, beadGateSeenUpdate(gate.AwaitID), actor); err != nil { + if err := store.UpdateIssue(ctx, gate.ID, beadGateSeenUpdate(gate.AwaitID), currentActor()); err != nil { return err } commandDidWrite.Store(true) @@ -1575,7 +1575,7 @@ func beadGateRetargetUpdate(awaitID string, dropSeen bool) map[string]interface{ // closeGate closes a gate issue with the given reason func closeGate(_ interface{}, gateID, reason string) error { - if err := store.CloseIssue(rootCtx, gateID, reason, actor, ""); err != nil { + if err := store.CloseIssue(rootCtx, gateID, reason, currentActor(), ""); err != nil { return err } commandDidWrite.Store(true) diff --git a/cmd/bd/gate_discover.go b/cmd/bd/gate_discover.go index f66c3d4fa..b1842e1de 100644 --- a/cmd/bd/gate_discover.go +++ b/cmd/bd/gate_discover.go @@ -581,7 +581,7 @@ func updateGateAwaitID(_ interface{}, gateID, runID string) error { updates := map[string]interface{}{ "await_id": runID, } - if err := store.UpdateIssue(rootCtx, gateID, updates, actor); err != nil { + if err := store.UpdateIssue(rootCtx, gateID, updates, currentActor()); err != nil { return err } return nil diff --git a/cmd/bd/gate_proxied_server.go b/cmd/bd/gate_proxied_server.go index 8597a749b..f83e66186 100644 --- a/cmd/bd/gate_proxied_server.go +++ b/cmd/bd/gate_proxied_server.go @@ -140,7 +140,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { } for gateID, runID := range discovered { - if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"await_id": runID}, actor); err != nil { + if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"await_id": runID}, currentActor()); err != nil { out.awaitErrs[gateID] = fmt.Errorf("failed to update gate with discovered run ID: %w", err) continue } @@ -168,7 +168,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { if before != nil && before.Status == types.StatusClosed { continue } - res, closeErr := uw.IssueUseCase().CloseIssue(ctx, r.gate.ID, domain.CloseIssueParams{Reason: r.reason}, actor) + res, closeErr := uw.IssueUseCase().CloseIssue(ctx, r.gate.ID, domain.CloseIssueParams{Reason: r.reason}, currentActor()) if closeErr != nil { out.closeErrs[r.gate.ID] = closeErr continue @@ -192,7 +192,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { } for _, c := range applied.closed { - audit.LogFieldChange(c.after.ID, "status", c.oldStatus, "closed", actor, c.reason) + audit.LogFieldChange(c.after.ID, "status", c.oldStatus, "closed", currentActor(), c.reason) } if len(applied.closed) > 0 || len(applied.updated) > 0 { commandDidWrite.Store(true) @@ -326,7 +326,7 @@ func runGateAddWaiterProxiedServer(_ *cobra.Command, ctx context.Context, args [ } newWaiters := append(issue.Waiters, waiter) - if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"waiters": newWaiters}, actor); err != nil { + if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"waiters": newWaiters}, currentActor()); err != nil { return out, "", fmt.Errorf("updating gate: %w", err) } if after, getErr := uw.IssueUseCase().GetIssue(ctx, gateID); getErr == nil { @@ -394,7 +394,7 @@ func runGateCreateProxiedServer(cmd *cobra.Command, ctx context.Context) error { } gate.Metadata = metadata - res, err := uw.IssueUseCase().CreateIssue(ctx, domain.CreateIssueParams{Issue: gate}, actor) + res, err := uw.IssueUseCase().CreateIssue(ctx, domain.CreateIssueParams{Issue: gate}, currentActor()) if err != nil { return out, "", fmt.Errorf("creating gate: %w", err) } @@ -404,7 +404,7 @@ func runGateCreateProxiedServer(cmd *cobra.Command, ctx context.Context) error { DependsOnID: res.Issue.ID, Type: types.DepBlocks, } - if err := uw.DependencyUseCase().AddDependency(ctx, dep, actor); err != nil { + if err := uw.DependencyUseCase().AddDependency(ctx, dep, currentActor()); err != nil { return out, "", fmt.Errorf("adding blocking dependency: %w", err) } @@ -464,7 +464,7 @@ func runGateResolveProxiedServer(cmd *cobra.Command, ctx context.Context, args [ return out, "", fmt.Errorf("%s is not a gate issue (type=%s)", gateID, issue.IssueType) } - res, err := uw.IssueUseCase().CloseIssue(ctx, gateID, domain.CloseIssueParams{Reason: reason}, actor) + res, err := uw.IssueUseCase().CloseIssue(ctx, gateID, domain.CloseIssueParams{Reason: reason}, currentActor()) if err != nil { return out, "", fmt.Errorf("closing gate: %w", err) } @@ -486,7 +486,7 @@ func runGateResolveProxiedServer(cmd *cobra.Command, ctx context.Context, args [ // double-resolve must not re-log it (same guard as the o.closed check in // close_proxied_server.go). if applied.closed && applied.after != nil { - audit.LogFieldChange(applied.after.ID, "status", applied.oldStatus, "closed", actor, reason) + audit.LogFieldChange(applied.after.ID, "status", applied.oldStatus, "closed", currentActor(), reason) } commandDidWrite.Store(true) diff --git a/cmd/bd/gc_proxied_server.go b/cmd/bd/gc_proxied_server.go index ed4818373..f5fbdb7df 100644 --- a/cmd/bd/gc_proxied_server.go +++ b/cmd/bd/gc_proxied_server.go @@ -93,7 +93,7 @@ func runGCProxiedServer(ctx context.Context) error { ids[i] = issue.ID } deleteResult, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (domain.DeleteIssuesResult, string, error) { - res, err := uw.IssueUseCase().DeleteIssues(ctx, domain.DeleteIssuesParams{IDs: ids}, actor) + res, err := uw.IssueUseCase().DeleteIssues(ctx, domain.DeleteIssuesParams{IDs: ids}, currentActor()) if err != nil { return domain.DeleteIssuesResult{}, "", err } diff --git a/cmd/bd/github.go b/cmd/bd/github.go index 77ee4e8e8..bedc3ef4b 100644 --- a/cmd/bd/github.go +++ b/cmd/bd/github.go @@ -419,7 +419,7 @@ func runGitHubSync(cmd *cobra.Command, args []string) error { } // Create the sync engine - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.OnMessage = func(msg string) { _, _ = fmt.Fprintln(out, " "+msg) } engine.OnWarning = func(msg string) { _, _ = fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } diff --git a/cmd/bd/gitlab.go b/cmd/bd/gitlab.go index 6657dcb3b..6bc94e0d0 100644 --- a/cmd/bd/gitlab.go +++ b/cmd/bd/gitlab.go @@ -511,7 +511,7 @@ func runGitLabSync(cmd *cobra.Command, args []string) error { } // Create the sync engine - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) if !jsonOutput { engine.OnMessage = func(msg string) { _, _ = fmt.Fprintln(out, " "+msg) } } diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 00faeb197..9a52f1415 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -1167,7 +1167,7 @@ func graphApplyEdgeBatchMetadata(edge GraphApplyEdge, depType types.DependencyTy } func executeGraphApply(ctx context.Context, plan *GraphApplyPlan, opts GraphApplyOptions) (*GraphApplyResult, error) { - req, err := buildGraphApplyBatchRequest(plan, opts, actor, getOwner()) + req, err := buildGraphApplyBatchRequest(plan, opts, currentActor(), getOwner()) if err != nil { return nil, err } diff --git a/cmd/bd/heartbeat.go b/cmd/bd/heartbeat.go index 281db80c2..6790047a7 100644 --- a/cmd/bd/heartbeat.go +++ b/cmd/bd/heartbeat.go @@ -68,11 +68,11 @@ Examples: defer result.Close() issueStore := result.Store - if err := issueStore.HeartbeatIssue(ctx, result.ResolvedID, actor); err != nil { + if err := issueStore.HeartbeatIssue(ctx, result.ResolvedID, currentActor()); err != nil { return HandleErrorRespectJSON("heartbeat %s: %v", result.ResolvedID, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "heartbeat", IssueIDs: []string{result.ResolvedID}, }); err != nil { @@ -94,7 +94,7 @@ func renderHeartbeatSuccess(id, title string) error { return outputJSON(map[string]string{ "id": id, "status": "heartbeat", - "owner": actor, + "owner": currentActor(), }) } fmt.Printf("%s Heartbeat %s (lease refreshed)\n", ui.RenderPass("✓"), formatFeedbackID(id, title)) diff --git a/cmd/bd/heartbeat_proxied_server.go b/cmd/bd/heartbeat_proxied_server.go index 80de4a033..c09772f52 100644 --- a/cmd/bd/heartbeat_proxied_server.go +++ b/cmd/bd/heartbeat_proxied_server.go @@ -41,7 +41,7 @@ func runHeartbeatProxiedServer(ctx context.Context, id string) error { } // Wisps resolve here too and are refused below: the repo verb // classifies them ErrNotClaimable ("is ephemeral"), same as classic. - if herr := uw.IssueUseCase().Heartbeat(ctx, issue.ID, actor); herr != nil { + if herr := uw.IssueUseCase().Heartbeat(ctx, issue.ID, currentActor()); herr != nil { return heartbeatProxiedOutcome{}, fmt.Errorf("heartbeat %s: %w", issue.ID, herr) } return heartbeatProxiedOutcome{id: issue.ID, title: issue.Title}, nil diff --git a/cmd/bd/human.go b/cmd/bd/human.go index c87faa2d6..19fb98907 100644 --- a/cmd/bd/human.go +++ b/cmd/bd/human.go @@ -307,12 +307,12 @@ Examples: warnIfNotHumanLabeled(issue) - _, err = targetStore.AddIssueComment(ctx, resolvedID, actor, commentText) + _, err = targetStore.AddIssueComment(ctx, resolvedID, currentActor(), commentText) if err != nil { return HandleErrorRespectJSON("adding comment: %v", err) } - if err := targetStore.CloseIssue(ctx, resolvedID, "Responded", actor, ""); err != nil { + if err := targetStore.CloseIssue(ctx, resolvedID, "Responded", currentActor(), ""); err != nil { return HandleErrorRespectJSON("closing bead: %v", err) } @@ -403,7 +403,7 @@ Examples: warnIfNotHumanLabeled(issue) - if err := targetStore.CloseIssue(ctx, resolvedID, closeReason, actor, ""); err != nil { + if err := targetStore.CloseIssue(ctx, resolvedID, closeReason, currentActor(), ""); err != nil { return HandleErrorRespectJSON("closing bead: %v", err) } diff --git a/cmd/bd/human_proxied_server.go b/cmd/bd/human_proxied_server.go index 10c948add..924ed2632 100644 --- a/cmd/bd/human_proxied_server.go +++ b/cmd/bd/human_proxied_server.go @@ -122,9 +122,9 @@ func closeHumanProxied(ctx context.Context, id, comment, closeReason, commitVerb if comment != "" { var cerr error if isWisp { - _, cerr = uw.CommentUseCase().AddCommentToWisp(ctx, issue.ID, actor, comment) + _, cerr = uw.CommentUseCase().AddCommentToWisp(ctx, issue.ID, currentActor(), comment) } else { - _, cerr = uw.CommentUseCase().AddCommentToIssue(ctx, issue.ID, actor, comment) + _, cerr = uw.CommentUseCase().AddCommentToIssue(ctx, issue.ID, currentActor(), comment) } if cerr != nil { return humanCloseResult{}, "", fmt.Errorf("adding comment: %w", cerr) @@ -133,9 +133,9 @@ func closeHumanProxied(ctx context.Context, id, comment, closeReason, commitVerb params := domain.CloseIssueParams{Reason: closeReason} if isWisp { - _, err = uw.IssueUseCase().CloseWisp(ctx, issue.ID, params, actor) + _, err = uw.IssueUseCase().CloseWisp(ctx, issue.ID, params, currentActor()) } else { - _, err = uw.IssueUseCase().CloseIssue(ctx, issue.ID, params, actor) + _, err = uw.IssueUseCase().CloseIssue(ctx, issue.ID, params, currentActor()) } if err != nil { return humanCloseResult{}, "", fmt.Errorf("closing bead: %w", err) diff --git a/cmd/bd/jira.go b/cmd/bd/jira.go index 099359b2a..b03574fe9 100644 --- a/cmd/bd/jira.go +++ b/cmd/bd/jira.go @@ -139,7 +139,7 @@ func runJiraSync(cmd *cobra.Command, args []string) error { return HandleErrorRespectJSON("initializing Jira tracker: %v", err) } - engine := tracker.NewEngine(jt, trackerStore, actor) + engine := tracker.NewEngine(jt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } diff --git a/cmd/bd/label.go b/cmd/bd/label.go index 46fcfd177..5f5a768db 100644 --- a/cmd/bd/label.go +++ b/cmd/bd/label.go @@ -165,7 +165,7 @@ func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, ope } } result, uerr := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: issueID, Patch: patch, }) @@ -361,7 +361,7 @@ func removeLabelsByPrefix(ctx context.Context, issueIDs []string, prefix string, outcomes := make([]labelEditOutcome, 0, len(targets)) for _, target := range targets { result, uerr := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: target.issueID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Remove: target.labels}}, }) @@ -707,7 +707,7 @@ var labelPropagateCmd = &cobra.Command{ commitMsg := fmt.Sprintf("bd: propagate label '%s' from %s to %d children", label, parentID, len(children)) err = transactHonoringAutoCommit(ctx, store, commitMsg, func(tx storage.Transaction) error { for _, child := range children { - if err := tx.AddLabel(ctx, child.ID, label, actor); err != nil { + if err := tx.AddLabel(ctx, child.ID, label, currentActor()); err != nil { return fmt.Errorf("add label '%s' on %s: %w", label, child.ID, err) } } @@ -809,7 +809,7 @@ func runLabelRename(ctx context.Context, args []string, dryRun bool) error { if usesProxiedServer() { renamed, merged, err = runLabelRenameProxiedServer(ctx, oldLabel, newLabel) } else { - renamed, merged, _, err = store.RenameLabel(ctx, oldLabel, newLabel, actor) + renamed, merged, _, err = store.RenameLabel(ctx, oldLabel, newLabel, currentActor()) } // Recorded from renamed>0 BEFORE the error check: a rename can commit its // SQL side and still return a non-nil err (e.g. the Dolt publication step diff --git a/cmd/bd/label_proxied_server.go b/cmd/bd/label_proxied_server.go index 9d5b955a1..0bf9215f0 100644 --- a/cmd/bd/label_proxied_server.go +++ b/cmd/bd/label_proxied_server.go @@ -148,7 +148,7 @@ func runLabelRenameProxiedServer(ctx context.Context, oldLabel, newLabel string) err = uow.RunTx(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (string, error) { var rerr error - renamed, merged, _, rerr = uw.LabelUseCase().RenameLabel(ctx, oldLabel, newLabel, actor) //nolint:forbidigo // bulk rename; no role serves it, and Patch.Labels would reintroduce the per-issue add/remove event divergence this delegation replaced + renamed, merged, _, rerr = uw.LabelUseCase().RenameLabel(ctx, oldLabel, newLabel, currentActor()) //nolint:forbidigo // bulk rename; no role serves it, and Patch.Labels would reintroduce the per-issue add/remove event divergence this delegation replaced if rerr != nil { return "", fmt.Errorf("rename label '%s' -> '%s': %w", oldLabel, newLabel, rerr) } @@ -209,9 +209,9 @@ func runLabelPropagateProxiedServer(ctx context.Context, args []string) error { for _, child := range children { var e error if child.Ephemeral { - e = uw.LabelUseCase().AddWispLabel(ctx, child.ID, label, actor) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor + e = uw.LabelUseCase().AddWispLabel(ctx, child.ID, label, currentActor()) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor } else { - e = uw.LabelUseCase().AddLabel(ctx, child.ID, label, actor) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor + e = uw.LabelUseCase().AddLabel(ctx, child.ID, label, currentActor()) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor } if e != nil { return "", fmt.Errorf("add label '%s' on %s: %w", label, child.ID, e) diff --git a/cmd/bd/linear.go b/cmd/bd/linear.go index 8e24cb777..f6ba42990 100644 --- a/cmd/bd/linear.go +++ b/cmd/bd/linear.go @@ -338,14 +338,14 @@ func runLinearSync(cmd *cobra.Command, args []string) error { } } - engine := tracker.NewEngine(lt, trackerStore, actor) + engine := tracker.NewEngine(lt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PullHooks = buildLinearPullHooksForStore(ctx, trackerStore, linearPullHookOptions{ Milestones: milestones, DryRun: dryRun, - Actor: actor, + Actor: currentActor(), }) opts := tracker.SyncOptions{ @@ -488,7 +488,7 @@ func buildLinearPullHooksForStore(ctx context.Context, st tracker.Store, opts li hooks := &tracker.PullHooks{} hookActor := opts.Actor if hookActor == "" { - hookActor = actor + hookActor = currentActor() } var generateID func(context.Context, *types.Issue) error diff --git a/cmd/bd/link.go b/cmd/bd/link.go index 772a88d91..a05be924b 100644 --- a/cmd/bd/link.go +++ b/cmd/bd/link.go @@ -77,13 +77,13 @@ Examples: Type: dt, } - if err := fromStore.AddDependencyWithOptions(ctx, dep, actor, storage.DependencyAddOptions{EmitEvent: true}); err != nil { + if err := fromStore.AddDependencyWithOptions(ctx, dep, currentActor(), storage.DependencyAddOptions{EmitEvent: true}); err != nil { return HandleErrorRespectJSON("%v", err) } warnIfCyclesExist(fromStore) - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "link", IssueIDs: []string{fromID, toID}, }); err != nil { diff --git a/cmd/bd/link_proxied_server.go b/cmd/bd/link_proxied_server.go index aa7fac1b9..aef888968 100644 --- a/cmd/bd/link_proxied_server.go +++ b/cmd/bd/link_proxied_server.go @@ -35,7 +35,7 @@ func runLinkProxiedServer(cmd *cobra.Command, ctx context.Context, args []string // Source-routed, like the direct twin's store.AddDependencyWithOptions: // `bd link` takes whatever id the caller names, and a wisp source has no // row in the issues plane for the edge to hang off. - if _, err := uw.DependencyUseCase().AddDependencies(ctx, []*types.Dependency{dep}, actor, domain.BulkAddDepsOpts{}); err != nil { + if _, err := uw.DependencyUseCase().AddDependencies(ctx, []*types.Dependency{dep}, currentActor(), domain.BulkAddDepsOpts{}); err != nil { return "", err } return fmt.Sprintf("bd: link %s %s", id1, id2), nil diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 9fd38b3b9..bdcb83bd1 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -893,11 +893,74 @@ func resolveConfiguredActor() string { return config.GetString("actor") } +// actorGitFallbackPending records that PersistentPreRunE deferred the +// getActorWithGit fallback (git config user.name, then $USER) instead of +// resolving it eagerly. Most invocations — every read-only list/show/query a +// tool like gascity issues by the hundred — never consult the actor, and the +// eager lookup cost one `git config` subprocess per bd call. currentActor +// performs the deferred resolution on first use, so the resolved value and its +// priority order are unchanged; only the moment of the lookup moves. +// +// Guarded by actorMu together with the resolution it triggers. Plain writes to +// actor elsewhere stay unguarded, exactly as before this was introduced. +var ( + actorMu sync.Mutex + actorGitFallbackPending bool +) + +// deferActorGitFallback replaces the former eager `actor = getActorWithGit()` +// in PersistentPreRunE. A non-empty actor (--actor, BEADS_ACTOR, BD_ACTOR or +// config.yaml) is already the answer getActorWithGit would return, so only an +// empty actor needs the deferred fallback. +func deferActorGitFallback() { + actorMu.Lock() + defer actorMu.Unlock() + actorGitFallbackPending = actor == "" +} + +// currentActor returns the actor for this invocation, resolving a fallback +// deferred by deferActorGitFallback on first use. Command code reads the actor +// through this instead of the raw global so that the git lookup happens only +// when an actor is actually needed. +func currentActor() string { + actorMu.Lock() + defer actorMu.Unlock() + resolvePendingActorLocked() + return actor +} + +// resolvePendingActorLocked performs a deferred fallback. Caller holds actorMu. +func resolvePendingActorLocked() { + if !actorGitFallbackPending { + return + } + actorGitFallbackPending = false + deferred := actor + actor = resolveActorWithGit() + // Keep the CommandContext copy in step, as the eager assignment did + // before syncCommandContext copied it. + if cmdCtx != nil && cmdCtx.Actor == deferred { + cmdCtx.Actor = actor + } +} + // getActorWithGit returns the actor for audit trails with git config fallback. // Priority: --actor flag > BEADS_ACTOR env > BD_ACTOR env (deprecated) > git config user.name > $USER > "unknown" // This provides a sensible default for developers: their git identity is used unless -// explicitly overridden +// explicitly overridden. +// +// After PersistentPreRunE it returns the same value as currentActor (resolving +// a deferred fallback once, into the global, as the eager assignment did). func getActorWithGit() string { + actorMu.Lock() + defer actorMu.Unlock() + resolvePendingActorLocked() + return resolveActorWithGit() +} + +// resolveActorWithGit is getActorWithGit's resolution order, without the +// deferred-fallback bookkeeping. Caller holds actorMu or owns the global. +func resolveActorWithGit() string { // If actor is already set (from --actor flag), use it if actor != "" { return actor @@ -1674,11 +1737,14 @@ var rootCmd = &cobra.Command{ return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapReadonly)) } - // Set actor for audit trail - actor = getActorWithGit() - // Attach actor to the command span now that we have it. - if commandSpan != nil { - commandSpan.SetAttributes(attribute.String("bd.actor", actor)) + // Set actor for audit trail. The git config user.name fallback is + // resolved lazily by currentActor, so commands that never consult the + // actor (read-only queries) do not spawn a git subprocess for it. + deferActorGitFallback() + // Attach actor to the command span. Only a recording span (telemetry + // enabled) needs the value, so a noop span does not force the lookup. + if commandSpan != nil && commandSpan.IsRecording() { + commandSpan.SetAttributes(attribute.String("bd.actor", currentActor())) } // Check if this is a read-only command (GH#804) or an explicitly diff --git a/cmd/bd/markdown.go b/cmd/bd/markdown.go index 2c002ad74..b902ad7a4 100644 --- a/cmd/bd/markdown.go +++ b/cmd/bd/markdown.go @@ -425,8 +425,8 @@ func markdownBatchActor(in createInput) string { if in.createdBy != "" { return in.createdBy } - if actor != "" { - return actor + if currentActor() != "" { + return currentActor() } return "bd" } diff --git a/cmd/bd/merge_slot.go b/cmd/bd/merge_slot.go index 927327472..cfd715394 100644 --- a/cmd/bd/merge_slot.go +++ b/cmd/bd/merge_slot.go @@ -126,7 +126,7 @@ func runMergeSlotCreate(cmd *cobra.Command, args []string) error { } }() - issue, err := store.MergeSlotCreate(rootCtx, actor) + issue, err := store.MergeSlotCreate(rootCtx, currentActor()) if err != nil { return HandleErrorRespectJSON("%v", err) } @@ -222,13 +222,13 @@ func runMergeSlotAcquire(cmd *cobra.Command, args []string) error { holder := mergeSlotHolder if holder == "" { - holder = actor + holder = currentActor() } if holder == "" { return HandleError("no holder specified; use --holder or set BEADS_ACTOR env var") } - result, err := store.MergeSlotAcquire(rootCtx, holder, actor, mergeSlotAddWaiter) + result, err := store.MergeSlotAcquire(rootCtx, holder, currentActor(), mergeSlotAddWaiter) if err != nil { return HandleErrorRespectJSON("%v", err) } @@ -305,7 +305,7 @@ func runMergeSlotRelease(cmd *cobra.Command, args []string) error { } }() - if err := store.MergeSlotRelease(rootCtx, mergeSlotHolder, actor); err != nil { + if err := store.MergeSlotRelease(rootCtx, mergeSlotHolder, currentActor()); err != nil { return HandleErrorRespectJSON("%v", err) } diff --git a/cmd/bd/migrate_issues.go b/cmd/bd/migrate_issues.go index 382d7ac3b..192afcaf9 100644 --- a/cmd/bd/migrate_issues.go +++ b/cmd/bd/migrate_issues.go @@ -600,7 +600,7 @@ func executeMigration(ctx context.Context, s storage.DoltStorage, migrationSet [ for _, id := range migrationSet { if err := tx.UpdateIssue(ctx, id, map[string]interface{}{ "source_repo": to, - }, actor); err != nil { + }, currentActor()); err != nil { return fmt.Errorf("failed to update issue %s: %w", id, err) } } diff --git a/cmd/bd/mol_bond.go b/cmd/bd/mol_bond.go index 3ea600de9..4f8a5f2c2 100644 --- a/cmd/bd/mol_bond.go +++ b/cmd/bd/mol_bond.go @@ -148,21 +148,21 @@ func runMolBond(cmd *cobra.Command, args []string) error { var result *BondResult switch { case aIsProto && bIsProto: - result, err = bondProtoProto(ctx, store, issueA, issueB, in.bondType, in.customTitle, actor) + result, err = bondProtoProto(ctx, store, issueA, issueB, in.bondType, in.customTitle, currentActor()) case aIsProto && !bIsProto: if cookedA { - result, err = bondProtoMolWithSubgraph(ctx, store, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolWithSubgraph(ctx, store, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } else { - result, err = bondProtoMol(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMol(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } case !aIsProto && bIsProto: if cookedB { - result, err = bondProtoMolWithSubgraph(ctx, store, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolWithSubgraph(ctx, store, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } else { - result, err = bondMolProto(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondMolProto(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } default: - result, err = bondMolMol(ctx, store, issueA, issueB, in.bondType, actor) + result, err = bondMolMol(ctx, store, issueA, issueB, in.bondType, currentActor()) } if err != nil { return HandleErrorRespectJSON("bonding: %v", err) diff --git a/cmd/bd/mol_burn.go b/cmd/bd/mol_burn.go index 3062ff794..48293d832 100644 --- a/cmd/bd/mol_burn.go +++ b/cmd/bd/mol_burn.go @@ -190,7 +190,7 @@ func burnMultipleMolecules(ctx context.Context, moleculeIDs []string, dryRun, fo // Batch delete all wisps in one call if len(wispIDs) > 0 { - result, err := burnWisps(ctx, store, wispIDs, actor) + result, err := burnWisps(ctx, store, wispIDs, currentActor()) if err != nil { if !jsonOutput { fmt.Fprintf(os.Stderr, "Error burning wisps: %v\n", err) @@ -296,7 +296,7 @@ func burnWispMolecule(ctx context.Context, resolvedID string, dryRun, force bool } } - result, err := burnWisps(ctx, store, wispIDs, actor) + result, err := burnWisps(ctx, store, wispIDs, currentActor()) if err != nil { return HandleErrorRespectJSON("burning wisp: %v", err) } diff --git a/cmd/bd/mol_current.go b/cmd/bd/mol_current.go index f3d92d33f..6769f01ef 100644 --- a/cmd/bd/mol_current.go +++ b/cmd/bd/mol_current.go @@ -73,7 +73,7 @@ Use --limit or --range to view specific steps: agent := forAgent if agent == "" { - agent = actor + agent = currentActor() } if usesProxiedServer() { diff --git a/cmd/bd/mol_progress.go b/cmd/bd/mol_progress.go index ba35c2b61..df77f9359 100644 --- a/cmd/bd/mol_progress.go +++ b/cmd/bd/mol_progress.go @@ -58,7 +58,7 @@ Example: } moleculeID = resolved } else { - moleculeIDs := findInProgressMoleculeIDs(ctx, store, actor) + moleculeIDs := findInProgressMoleculeIDs(ctx, store, currentActor()) if len(moleculeIDs) == 0 { if jsonOutput { return outputJSON([]interface{}{}) diff --git a/cmd/bd/mol_proxied_server.go b/cmd/bd/mol_proxied_server.go index 6136499dc..04f16955e 100644 --- a/cmd/bd/mol_proxied_server.go +++ b/cmd/bd/mol_proxied_server.go @@ -124,7 +124,7 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { spawnResult, err := cloneSubgraphInto(ctx, w, subgraph, CloneOptions{ Vars: vars, Assignee: in.assignee, - Actor: actor, + Actor: currentActor(), Prefix: types.IDPrefixMol, }) if err != nil { @@ -138,7 +138,7 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { return pourProxiedResult{}, "", fmt.Errorf("loading spawned mol: %w", err) } for _, attach := range attachments { - bondResult, err := bondProtoMolAttachInto(ctx, w, attach.subgraph, attach.issue, spawnedMol, in.attachType, vars, "", actor, false, true) + bondResult, err := bondProtoMolAttachInto(ctx, w, attach.subgraph, attach.issue, spawnedMol, in.attachType, vars, "", currentActor(), false, true) if err != nil { return pourProxiedResult{}, "", fmt.Errorf("attaching %s: %w", attach.id, err) } @@ -278,7 +278,7 @@ func runMolProgressProxiedServer(ctx context.Context, args []string) error { } moleculeID = resolved } else { - moleculeIDs := findInProgressMoleculeIDs(ctx, r, actor) + moleculeIDs := findInProgressMoleculeIDs(ctx, r, currentActor()) if len(moleculeIDs) == 0 { if jsonOutput { return outputJSON([]interface{}{}) @@ -426,13 +426,13 @@ func runMolBondProxiedServer(ctx context.Context, in molBondInput) error { var result *BondResult switch { case aIsProto && bIsProto: - result, err = bondProtoProtoInto(ctx, w, issueA, issueB, in.bondType, in.customTitle, actor) + result, err = bondProtoProtoInto(ctx, w, issueA, issueB, in.bondType, in.customTitle, currentActor()) case aIsProto && !bIsProto: - result, err = bondProtoMolAttachInto(ctx, w, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolAttachInto(ctx, w, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) case !aIsProto && bIsProto: - result, err = bondProtoMolAttachInto(ctx, w, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolAttachInto(ctx, w, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) default: - result, err = bondMolMolInto(ctx, w, issueA, issueB, in.bondType, actor) + result, err = bondMolMolInto(ctx, w, issueA, issueB, in.bondType, currentActor()) } if err != nil { return bondTxResult{}, "", fmt.Errorf("bonding: %w", err) @@ -501,7 +501,7 @@ func runMolSquashProxiedServer(ctx context.Context, in molSquashInput) error { return &SquashResult{MoleculeID: moleculeID, SquashedCount: 0}, "", nil } - result, err := squashMoleculeInto(ctx, w, subgraph.Root, wispChildren, in.keepChildren, in.summary, actor) + result, err := squashMoleculeInto(ctx, w, subgraph.Root, wispChildren, in.keepChildren, in.summary, currentActor()) if err != nil { return nil, "", fmt.Errorf("squashing molecule: %w", err) } @@ -638,7 +638,7 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b if len(wispIDs) > 0 { // The wisp half's direct twin is burnWisps -> `transact`. result, err := uow.RunTxResult(explicitCommitPointContext(ctx), uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*BurnResult, string, error) { - r, err := burnWispsInto(ctx, newUOWMolWriter(uw), wispIDs, actor) + r, err := burnWispsInto(ctx, newUOWMolWriter(uw), wispIDs, currentActor()) if err != nil { return nil, "", err } @@ -672,7 +672,7 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b IDs: issueIDs, Cascade: true, UpdateTextReferences: true, - }, actor) + }, currentActor()) if err != nil { return BurnResult{}, "", fmt.Errorf("burning %s: %w", id, err) } diff --git a/cmd/bd/mol_squash.go b/cmd/bd/mol_squash.go index 223e31955..39363cbdd 100644 --- a/cmd/bd/mol_squash.go +++ b/cmd/bd/mol_squash.go @@ -178,7 +178,7 @@ func runMolSquash(cmd *cobra.Command, args []string) error { return nil } - result, err := squashMolecule(ctx, store, subgraph.Root, wispChildren, in.keepChildren, in.summary, actor) + result, err := squashMolecule(ctx, store, subgraph.Root, wispChildren, in.keepChildren, in.summary, currentActor()) if err != nil { return HandleErrorRespectJSON("squashing molecule: %v", err) } diff --git a/cmd/bd/mutate_proxied_server.go b/cmd/bd/mutate_proxied_server.go index 7008e9a41..fa72c238e 100644 --- a/cmd/bd/mutate_proxied_server.go +++ b/cmd/bd/mutate_proxied_server.go @@ -54,15 +54,15 @@ func proxiedUpdateIssueFields(ctx context.Context, id, commitMsg string, updates // bd-98s5c: an unguarded assignee update (bd assign via the proxied // server) must not silently overwrite another actor's live claim. if newAssignee, ok := updates["assignee"].(string); ok { - if err := validateIssueReassignable(id, issue, actor, newAssignee, + if err := validateIssueReassignable(id, issue, currentActor(), newAssignee, uowClaimPoolAliases(ctx, uw), force); err != nil { return err } } if isWisp { - return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, currentActor()) } - return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, currentActor()) }) } @@ -95,7 +95,7 @@ func proxiedAssign(ctx context.Context, id, assignee string, force bool, ifRevis // from the guarded write below instead of this policy refusal — see // ifRevisionAlreadyStale's doc. if !ifRevisionAlreadyStale(current, ifRevision) { - if verr := validateIssueReassignable(id, current, actor, assignee, + if verr := validateIssueReassignable(id, current, currentActor(), assignee, uowClaimPoolAliases(ctx, uw), force); verr != nil { return struct{}{}, verr } @@ -111,7 +111,7 @@ func proxiedAssign(ctx context.Context, id, assignee string, force bool, ifRevis return nil, err } result, err := runCommandUpdateMutation(ctx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: id, patch: issueops.IssuePatch{ Assignee: issueops.Field[string]{Set: true, Value: assignee}, @@ -183,9 +183,9 @@ func runNoteProxiedServer(ctx context.Context, id, noteText string) error { combined += noteText updates := map[string]any{"notes": combined} if isWisp { - return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, currentActor()) } - return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, currentActor()) }) if err != nil { return HandleErrorRespectJSON("note %s: %v", id, err) @@ -245,7 +245,7 @@ func runTagProxiedServer(ctx context.Context, id, label string) error { return HandleErrorRespectJSON("tag %s: %v", id, err) } result, err := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: details.ID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Add: []string{label}}}, }) diff --git a/cmd/bd/note.go b/cmd/bd/note.go index b0c8eeeab..9ee62cb9e 100644 --- a/cmd/bd/note.go +++ b/cmd/bd/note.go @@ -173,10 +173,10 @@ To read notes on an issue, use: bd show `, updates := map[string]interface{}{ issueops.OpAppendNotes: noteText, } - if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, actor); err != nil { + if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, currentActor()); err != nil { return HandleErrorRespectJSON("updating %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "note", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/notion.go b/cmd/bd/notion.go index 664cd31f6..3ec42b336 100644 --- a/cmd/bd/notion.go +++ b/cmd/bd/notion.go @@ -467,7 +467,7 @@ func runNotionSync(cmd *cobra.Command, _ []string) error { return HandleError("initializing Notion tracker: %v", err) } - engine := tracker.NewEngine(nt, store, actor) + engine := tracker.NewEngine(nt, store, currentActor()) engine.PullHooks = buildNotionPullHooks(ctx) unsupportedStats := newNotionUnsupportedPushStats() engine.PushHooks = buildNotionPushHooks(ctx, nt, unsupportedStats) diff --git a/cmd/bd/pour.go b/cmd/bd/pour.go index 91aa946eb..7260365ee 100644 --- a/cmd/bd/pour.go +++ b/cmd/bd/pour.go @@ -198,7 +198,7 @@ func runPour(cmd *cobra.Command, args []string) error { return nil } - result, err := spawnMolecule(ctx, store, subgraph, vars, in.assignee, actor, false, types.IDPrefixMol) + result, err := spawnMolecule(ctx, store, subgraph, vars, in.assignee, currentActor(), false, types.IDPrefixMol) if err != nil { return HandleError("pouring proto: %v", err) } @@ -211,7 +211,7 @@ func runPour(cmd *cobra.Command, args []string) error { } for _, attach := range attachments { - bondResult, err := bondProtoMol(ctx, store, attach.issue, spawnedMol, in.attachType, vars, "", actor, false, true) + bondResult, err := bondProtoMol(ctx, store, attach.issue, spawnedMol, in.attachType, vars, "", currentActor(), false, true) if err != nil { return HandleError("attaching %s: %v", attach.id, err) } diff --git a/cmd/bd/priority.go b/cmd/bd/priority.go index 481afecec..66a633e71 100644 --- a/cmd/bd/priority.go +++ b/cmd/bd/priority.go @@ -78,10 +78,10 @@ Examples: updates := map[string]interface{}{ "priority": priority, } - if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, actor); err != nil { + if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, currentActor()); err != nil { return HandleErrorRespectJSON("updating %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "priority", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/promote.go b/cmd/bd/promote.go index b1238abbf..44a42eea0 100644 --- a/cmd/bd/promote.go +++ b/cmd/bd/promote.go @@ -97,13 +97,13 @@ Examples: return HandleErrorRespectJSON("%s is not a wisp (already persistent)", fullID) } - if err := store.PromoteFromEphemeral(ctx, fullID, actor); err != nil { + if err := store.PromoteFromEphemeral(ctx, fullID, currentActor()); err != nil { return HandleErrorRespectJSON("promoting %s: %v", fullID, err) } // Add promotion comment (issue is now in permanent table, AddComment routes correctly // via GetIssue fallback) - if err := store.AddComment(ctx, fullID, actor, promotionComment(reason)); err != nil { + if err := store.AddComment(ctx, fullID, currentActor(), promotionComment(reason)); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to add promotion comment to %s: %v\n", fullID, err) } diff --git a/cmd/bd/promote_proxied_server.go b/cmd/bd/promote_proxied_server.go index edadce2d5..38f6fb547 100644 --- a/cmd/bd/promote_proxied_server.go +++ b/cmd/bd/promote_proxied_server.go @@ -62,10 +62,10 @@ func runPromoteProxiedServer(ctx context.Context, id, reason string) error { } err = uow.RunTx(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (string, error) { - if err := uw.IssueUseCase().PromoteWisp(ctx, fullID, actor); err != nil { + if err := uw.IssueUseCase().PromoteWisp(ctx, fullID, currentActor()); err != nil { return "", err } - if _, err := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, actor, promotionComment(reason)); err != nil { + if _, err := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, currentActor(), promotionComment(reason)); err != nil { return "", fmt.Errorf("adding promotion comment: %w", err) } return fmt.Sprintf("bd: promote %s", fullID), nil diff --git a/cmd/bd/purge.go b/cmd/bd/purge.go index 0337db793..cead0ae98 100644 --- a/cmd/bd/purge.go +++ b/cmd/bd/purge.go @@ -165,7 +165,7 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { } request := issueops.SweepRequest{ - Actor: actor, + Actor: currentActor(), Tier: scope.tier, IDPattern: pattern, ProtectReferenced: scope.protectReferenced, diff --git a/cmd/bd/quick.go b/cmd/bd/quick.go index a0cdbf550..e9ab99c03 100644 --- a/cmd/bd/quick.go +++ b/cmd/bd/quick.go @@ -94,7 +94,7 @@ Example: // The issue and its parent-child edge commit in one transaction; a // failed edge rolls back the create instead of leaving a dep-less // child behind (same contract as bd create). - if err := createIssueWithDeps(ctx, store, issue, actor, createDepEdges{parentID: parentID}); err != nil { + if err := createIssueWithDeps(ctx, store, issue, currentActor(), createDepEdges{parentID: parentID}); err != nil { return HandleError("%v", err) } diff --git a/cmd/bd/quick_proxied_server.go b/cmd/bd/quick_proxied_server.go index 3035863bb..46ab79c62 100644 --- a/cmd/bd/quick_proxied_server.go +++ b/cmd/bd/quick_proxied_server.go @@ -47,7 +47,7 @@ func runQuickProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin Labels: labels, InheritLabelsFromParent: parentID != "", } - result, err := uw.IssueUseCase().CreateIssue(ctx, params, actor) + result, err := uw.IssueUseCase().CreateIssue(ctx, params, currentActor()) if err != nil { return nil, "", err } diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index babcefa35..7e7e8aacf 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -159,7 +159,7 @@ This is useful for agents executing molecules to see which steps can run next.`, return nil } claimed := res.Claimed - if err := commitPendingIfEmbedded(ctx, activeStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, activeStore, currentActor(), doltAutoCommitParams{ Command: "ready", IssueIDs: []string{claimed.ID}, }); err != nil { diff --git a/cmd/bd/ready_input.go b/cmd/bd/ready_input.go index 0643ae3bd..e55a830c4 100644 --- a/cmd/bd/ready_input.go +++ b/cmd/bd/ready_input.go @@ -260,7 +260,7 @@ func readyRoleRequest(in readyInput) issueops.ReadyRequest { // claimNextRequest is `bd ready --claim`'s request to the ReadyClaimer role: // the shared ready question above, plus the claimant. func claimNextRequest(in readyInput) issueops.ClaimNextRequest { - return issueops.ClaimNextRequest{Actor: actor, Filter: readyRoleRequest(in)} + return issueops.ClaimNextRequest{Actor: currentActor(), Filter: readyRoleRequest(in)} } // briefModeConflict reports the usage error for a --brief combination that no diff --git a/cmd/bd/reclaim.go b/cmd/bd/reclaim.go index 34b40d98b..2a5262d99 100644 --- a/cmd/bd/reclaim.go +++ b/cmd/bd/reclaim.go @@ -104,7 +104,7 @@ Examples: } ctx := rootCtx - reclaimed, err := store.ReclaimExpiredLeases(ctx, olderThan, filter, actor) + reclaimed, err := store.ReclaimExpiredLeases(ctx, olderThan, filter, currentActor()) if err != nil { return HandleErrorRespectJSON("reclaim: %v", err) } @@ -113,7 +113,7 @@ Examples: for _, r := range reclaimed { ids = append(ids, r.ID) } - if err := commitPendingIfEmbedded(ctx, store, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, store, currentActor(), doltAutoCommitParams{ Command: "reclaim", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/relate.go b/cmd/bd/relate.go index 987ccdff9..f87f644d4 100644 --- a/cmd/bd/relate.go +++ b/cmd/bd/relate.go @@ -111,7 +111,7 @@ func runRelate(cmd *cobra.Command, args []string) error { } // bd relate is an explicit dependency verb, so it records history like // bd link / bd dep add (EmitEvent); only structural edge wiring stays silent. - if err := store.AddDependencyWithOptions(ctx, dep1, actor, storage.DependencyAddOptions{EmitEvent: true}); err != nil { + if err := store.AddDependencyWithOptions(ctx, dep1, currentActor(), storage.DependencyAddOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to add relates-to %s -> %s: %w", id1, id2, err) } // Add id2 -> id1 (bidirectional) @@ -120,7 +120,7 @@ func runRelate(cmd *cobra.Command, args []string) error { DependsOnID: id1, Type: types.DepRelatesTo, } - if err := store.AddDependencyWithOptions(ctx, dep2, actor, storage.DependencyAddOptions{EmitEvent: true}); err != nil { + if err := store.AddDependencyWithOptions(ctx, dep2, currentActor(), storage.DependencyAddOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to add relates-to %s -> %s: %w", id2, id1, err) } @@ -190,11 +190,11 @@ func runUnrelate(cmd *cobra.Command, args []string) error { // bd unrelate is an explicit dependency verb, so it records history // (EmitEvent) like bd dep remove; only structural teardown stays silent. // Remove id1 -> id2 - if err := store.RemoveDependencyWithOptions(ctx, id1, id2, actor, storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { + if err := store.RemoveDependencyWithOptions(ctx, id1, id2, currentActor(), storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to remove relates-to %s -> %s: %w", id1, id2, err) } // Remove id2 -> id1 (bidirectional) - if err := store.RemoveDependencyWithOptions(ctx, id2, id1, actor, storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { + if err := store.RemoveDependencyWithOptions(ctx, id2, id1, currentActor(), storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to remove relates-to %s -> %s: %w", id2, id1, err) } diff --git a/cmd/bd/relate_proxied_server.go b/cmd/bd/relate_proxied_server.go index 601b1a30d..34a6d688f 100644 --- a/cmd/bd/relate_proxied_server.go +++ b/cmd/bd/relate_proxied_server.go @@ -66,7 +66,7 @@ func runRelateProxiedServer(ctx context.Context, args []string) error { } // bd relate is an explicit dependency verb, so the bulk path's // EmitEvent trail matches the direct route's history behavior. - if _, err := uw.DependencyUseCase().AddDependencies(ctx, deps, actor, domain.BulkAddDepsOpts{}); err != nil { + if _, err := uw.DependencyUseCase().AddDependencies(ctx, deps, currentActor(), domain.BulkAddDepsOpts{}); err != nil { return "", err } return fmt.Sprintf("bd: relate %s %s", id1, id2), nil @@ -111,11 +111,11 @@ func runUnrelateProxiedServer(ctx context.Context, args []string) error { return "", fmt.Errorf("issue not found: %s", id) } } - removed1, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id1, id2, actor) + removed1, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id1, id2, currentActor()) if err != nil { return "", fmt.Errorf("failed to remove relates-to %s -> %s: %w", id1, id2, err) } - removed2, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id2, id1, actor) + removed2, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id2, id1, currentActor()) if err != nil { return "", fmt.Errorf("failed to remove relates-to %s -> %s: %w", id2, id1, err) } diff --git a/cmd/bd/rename_prefix.go b/cmd/bd/rename_prefix.go index 0ff1c8c91..68936732b 100644 --- a/cmd/bd/rename_prefix.go +++ b/cmd/bd/rename_prefix.go @@ -112,7 +112,7 @@ NOTE: This is a rare operation. Most users never need this command.`, ) } - if err := repairPrefixes(ctx, store, actor, newPrefix, issues, prefixes, dryRun); err != nil { + if err := repairPrefixes(ctx, store, currentActor(), newPrefix, issues, prefixes, dryRun); err != nil { return HandleError("failed to repair prefixes: %v", err) } if !dryRun { @@ -456,7 +456,7 @@ func renamePrefixInDB(ctx context.Context, oldPrefix, newPrefix string, issues [ continue } - if err := renameIssueKeepingBeadGates(ctx, store, issue, newID, gates[oldID], actor); err != nil { + if err := renameIssueKeepingBeadGates(ctx, store, issue, newID, gates[oldID], currentActor()); err != nil { return fmt.Errorf("failed to update issue %s: %w", oldID, err) } } diff --git a/cmd/bd/reopen.go b/cmd/bd/reopen.go index 6bdb3f7cd..6a6635e23 100644 --- a/cmd/bd/reopen.go +++ b/cmd/bd/reopen.go @@ -96,7 +96,7 @@ another actor won the race, so retrying the same guard is pointless).`, continue } reopened, err := ops.Reopen(opsCtx, issueops.ReopenRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fullID, Reason: reason, ExpectedVersion: ifRevision, @@ -150,7 +150,7 @@ another actor won the race, so retrying the same guard is pointless).`, } for s, ids := range mutatedStores { - if err := commitPendingIfEmbedded(ctx, s, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, s, currentActor(), doltAutoCommitParams{ Command: "reopen", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/reopen_proxied_server.go b/cmd/bd/reopen_proxied_server.go index a9fadd638..e61f85a22 100644 --- a/cmd/bd/reopen_proxied_server.go +++ b/cmd/bd/reopen_proxied_server.go @@ -66,7 +66,7 @@ func runReopenProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri reopenedIssues := []*types.Issue{} for _, target := range targets { result, err := lifecycle.Reopen(ctx, issueops.ReopenRequest{ - Actor: actor, + Actor: currentActor(), IssueID: target.id, Reason: reason, ExpectedVersion: ifRevision, @@ -94,7 +94,7 @@ func runReopenProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri continue } - audit.LogFieldChange(target.id, "status", string(target.status), string(types.StatusOpen), actor, reason) + audit.LogFieldChange(target.id, "status", string(target.status), string(types.StatusOpen), currentActor(), reason) if jsonOut { if issue := result.Issue; issue != nil { // `bd reopen` has never printed dependency records, on either diff --git a/cmd/bd/ship.go b/cmd/bd/ship.go index a8c0908ed..9e56be5bd 100644 --- a/cmd/bd/ship.go +++ b/cmd/bd/ship.go @@ -127,7 +127,7 @@ func runShip(cmd *cobra.Command, args []string) error { return nil } - if err := store.AddLabel(ctx, issue.ID, providesLabel, actor); err != nil { + if err := store.AddLabel(ctx, issue.ID, providesLabel, currentActor()); err != nil { return HandleErrorRespectJSON("adding label: %v", err) } diff --git a/cmd/bd/startup_git_exec_embedded_test.go b/cmd/bd/startup_git_exec_embedded_test.go new file mode 100644 index 000000000..4073da792 --- /dev/null +++ b/cmd/bd/startup_git_exec_embedded_test.go @@ -0,0 +1,141 @@ +//go:build cgo && linux + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// gitExecRecorder puts a `git` shim first on PATH that logs each invocation's +// arguments and then runs the real git, so a test can count the git processes +// a bd subprocess spawns. +type gitExecRecorder struct { + shimDir, log string +} + +func newGitExecRecorder(t *testing.T) *gitExecRecorder { + t.Helper() + realGit, err := exec.LookPath("git") + if err != nil { + t.Skipf("git not found: %v", err) + } + r := &gitExecRecorder{shimDir: t.TempDir()} + r.log = filepath.Join(r.shimDir, "git-calls.log") + script := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> '" + r.log + "'\nexec '" + realGit + "' \"$@\"\n" + if err := os.WriteFile(filepath.Join(r.shimDir, "git"), []byte(script), 0o755); err != nil { // #nosec G306 -- test shim must be executable + t.Fatal(err) + } + return r +} + +// run runs bd in dir with the shim on PATH and returns stdout plus the git +// invocations it made. +func (r *gitExecRecorder) run(t *testing.T, bd, dir string, args ...string) (string, []string) { + t.Helper() + if err := os.Remove(r.log); err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + cmd := exec.Command(bd, args...) + cmd.Dir = dir + env := envWithout(envWithout(bdEnv(dir), "BD_ACTOR"), "PATH") + for _, name := range []string{"GIT_DIR", "GIT_WORK_TREE", "GIT_COMMON_DIR", "GIT_CEILING_DIRECTORIES"} { + env = envWithout(env, name) + } + // backup.enabled is set explicitly: left at its default, embedded-mode + // auto-backup decides per command whether a git remote exists, which is + // a post-command probe of its own and not the startup path pinned here. + env = envWithout(envWithout(env, "BD_BACKUP_ENABLED"), "BEADS_BACKUP_ENABLED") + cmd.Env = append(env, "BD_BACKUP_ENABLED=false", + "PATH="+r.shimDir+string(os.PathListSeparator)+os.Getenv("PATH")) + out, err := cmd.Output() + if err != nil { + var stderr string + if exit, ok := err.(*exec.ExitError); ok { + stderr = string(exit.Stderr) + } + t.Fatalf("bd %s: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), err, out, stderr) + } + data, err := os.ReadFile(r.log) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + var calls []string + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line != "" { + calls = append(calls, line) + } + } + return string(out), calls +} + +// TestEmbeddedReadOnlyCommandsSpawnNoGit pins the startup cost tools that call +// bd by the hundred depend on: a read-only command resolves the repository +// in-process and never needs the actor, so it runs no git at all — from the +// main checkout and from a linked worktree sharing its database. A write still +// resolves the actor from git config user.name, lazily and to the same value. +func TestEmbeddedReadOnlyCommandsSpawnNoGit(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "gx") + issue := bdCreate(t, bd, dir, "startup git probe") + + worktree := filepath.Join(filepath.Dir(dir), filepath.Base(dir)+"-wt") + t.Cleanup(func() { _ = os.RemoveAll(worktree) }) + noHooks := "core.hooksPath=" + t.TempDir() // keep bd's installed hooks out of fixture setup + for _, args := range [][]string{ + {"-c", noHooks, "commit", "--allow-empty", "--no-verify", "-m", "base"}, + {"-c", noHooks, "worktree", "add", "-q", worktree}, + } { + cmd := gitCommand(args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + } + + rec := newGitExecRecorder(t) + for _, location := range []struct{ name, dir string }{ + {"main checkout", dir}, + {"linked worktree", worktree}, + } { + for _, args := range [][]string{ + {"list", "--json"}, + {"show", issue.ID, "--json"}, + } { + out, calls := rec.run(t, bd, location.dir, args...) + if !strings.Contains(out, issue.ID) { + t.Fatalf("%s: bd %s did not find %s:\n%s", location.name, args[0], issue.ID, out) + } + if len(calls) != 0 { + t.Errorf("%s: bd %s spawned %d git process(es), want 0:\n%s", + location.name, args[0], len(calls), strings.Join(calls, "\n")) + } + } + } + + // Positive control for the shim, and the deferred actor's value: a write + // asks git for user.name exactly once and records it as before. + out, calls := rec.run(t, bd, dir, "create", "--json", "lazy actor") + created := parseIssueJSON(t, []byte(out)) + if created.CreatedBy != "Test" { + t.Errorf("created_by = %q, want git user.name %q", created.CreatedBy, "Test") + } + nameLookups := 0 + for _, call := range calls { + if call == "config user.name" { + nameLookups++ + } + } + if nameLookups != 1 { + t.Errorf("bd create ran `git config user.name` %d times, want 1; calls:\n%s", nameLookups, strings.Join(calls, "\n")) + } +} diff --git a/cmd/bd/state.go b/cmd/bd/state.go index a3ad36623..b689b0f28 100644 --- a/cmd/bd/state.go +++ b/cmd/bd/state.go @@ -226,7 +226,7 @@ func runSetState(ctx context.Context, issueID, dimension, newValue, reason strin // could not write, so an event that disagreed with its subject's plane // fails the command instead of silently landing in the wrong table. created, err := lifecycle.Create(ctx, issueops.CreateRequest{ - Actor: actor, + Actor: currentActor(), Issue: &types.Issue{ Title: eventTitle, Description: eventDesc, @@ -254,7 +254,7 @@ func runSetState(ctx context.Context, issueID, dimension, newValue, reason strin // failed. The plane needs no branch: IssuePlaneOnly stays false and the // role resolves it inside its own transaction. if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fullID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{ Add: []string{newLabel}, diff --git a/cmd/bd/status.go b/cmd/bd/status.go index 855f67679..f5deaac38 100644 --- a/cmd/bd/status.go +++ b/cmd/bd/status.go @@ -88,7 +88,7 @@ Examples: // --no-blocked is not consulted here, and never was: an // assignee-scoped summary computes both numbers by a route that has // no fast path (issueops.StatsReporter.AssigneeStats). - result, err = reporter.AssigneeStats(rootCtx, issueops.AssigneeStatsRequest{Assignee: actor}) + result, err = reporter.AssigneeStats(rootCtx, issueops.AssigneeStatsRequest{Assignee: currentActor()}) } else { result, err = reporter.Stats(rootCtx, issueops.StatsRequest{SkipBlocked: noBlocked}) if err == nil && noBlocked && result.Summary.BlockedIssues != nil { diff --git a/cmd/bd/swarm.go b/cmd/bd/swarm.go index 996eefc40..9d066a023 100644 --- a/cmd/bd/swarm.go +++ b/cmd/bd/swarm.go @@ -1001,10 +1001,10 @@ Examples: Status: types.StatusOpen, Priority: issue.Priority, IssueType: types.TypeEpic, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.CreateIssue(ctx, wrapperEpic, actor); err != nil { + if err := store.CreateIssue(ctx, wrapperEpic, currentActor()); err != nil { return HandleErrorRespectJSON("failed to create wrapper epic: %v", err) } @@ -1012,9 +1012,9 @@ Examples: IssueID: issue.ID, DependsOnID: wrapperEpic.ID, Type: types.DepParentChild, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.AddDependency(ctx, dep, actor); err != nil { + if err := store.AddDependency(ctx, dep, currentActor()); err != nil { return HandleErrorRespectJSON("failed to link issue to epic: %v", err) } @@ -1081,10 +1081,10 @@ Examples: IssueType: "molecule", MolType: types.MolTypeSwarm, Assignee: coordinator, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.CreateIssue(ctx, swarmMol, actor); err != nil { + if err := store.CreateIssue(ctx, swarmMol, currentActor()); err != nil { return HandleErrorRespectJSON("failed to create swarm molecule: %v", err) } @@ -1092,9 +1092,9 @@ Examples: IssueID: swarmMol.ID, DependsOnID: epicID, Type: types.DepRelatesTo, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.AddDependency(ctx, dep, actor); err != nil { + if err := store.AddDependency(ctx, dep, currentActor()); err != nil { return HandleErrorRespectJSON("failed to link swarm to epic: %v", err) } diff --git a/cmd/bd/sync_push_pull.go b/cmd/bd/sync_push_pull.go index 63a2263ff..0e6b289da 100644 --- a/cmd/bd/sync_push_pull.go +++ b/cmd/bd/sync_push_pull.go @@ -272,7 +272,7 @@ func runADOPush(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Azure DevOps tracker: %w", err) } - engine := tracker.NewEngine(at, store, actor) + engine := tracker.NewEngine(at, store, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -319,7 +319,7 @@ func runADOPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Azure DevOps tracker: %w", err) } - engine := tracker.NewEngine(at, store, actor) + engine := tracker.NewEngine(at, store, currentActor()) engine.PullHooks = buildADOPullHooks(ctx, at, false, false, new(int), engine.OnWarning) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -369,7 +369,7 @@ func runJiraPush(cmd *cobra.Command, args []string) error { return HandleError("initializing Jira tracker: %v", err) } - engine := tracker.NewEngine(jt, trackerStore, actor) + engine := tracker.NewEngine(jt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PushHooks = buildJiraPushHooksForStore(ctx, trackerStore) @@ -417,7 +417,7 @@ func runJiraPull(cmd *cobra.Command, args []string) error { return HandleError("initializing Jira tracker: %v", err) } - engine := tracker.NewEngine(jt, trackerStore, actor) + engine := tracker.NewEngine(jt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -487,7 +487,7 @@ func runLinearPush(cmd *cobra.Command, args []string) error { return HandleError("%v", err) } - engine := tracker.NewEngine(lt, trackerStore, actor) + engine := tracker.NewEngine(lt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PushHooks = buildLinearPushHooksForStore(ctx, trackerStore, lt, len(args) > 0) @@ -552,12 +552,12 @@ func runLinearPull(cmd *cobra.Command, args []string) error { return HandleError("initializing Linear tracker: %v", err) } - engine := tracker.NewEngine(lt, trackerStore, actor) + engine := tracker.NewEngine(lt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PullHooks = buildLinearPullHooksForStore(ctx, trackerStore, linearPullHookOptions{ DryRun: dryRun, - Actor: actor, + Actor: currentActor(), }) result, err := engine.Sync(ctx, tracker.SyncOptions{ @@ -606,7 +606,7 @@ func runGitHubPush(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing GitHub tracker: %w", err) } - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PushHooks = buildGitHubPushHooks(gt) @@ -668,7 +668,7 @@ func runGitHubPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing GitHub tracker: %w", err) } - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.PullHooks = buildGitHubPullHooks(ctx) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -719,7 +719,7 @@ func runGitLabPush(cmd *cobra.Command, args []string) error { } out := cmd.OutOrStdout() - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.OnMessage = func(msg string) { if !jsonOutput { fmt.Fprintln(out, " "+msg) @@ -821,7 +821,7 @@ func runGitLabPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing GitLab tracker: %w", err) } - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.PullHooks = buildGitLabPullHooks(ctx) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -875,7 +875,7 @@ func runNotionPush(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Notion tracker: %w", err) } - engine := tracker.NewEngine(nt, store, actor) + engine := tracker.NewEngine(nt, store, currentActor()) unsupportedStats := newNotionUnsupportedPushStats() engine.PushHooks = buildNotionPushHooks(ctx, nt, unsupportedStats) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } @@ -929,7 +929,7 @@ func runNotionPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Notion tracker: %w", err) } - engine := tracker.NewEngine(nt, store, actor) + engine := tracker.NewEngine(nt, store, currentActor()) engine.PullHooks = buildNotionPullHooks(ctx) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } diff --git a/cmd/bd/tag.go b/cmd/bd/tag.go index 096b5f66c..02e4db5de 100644 --- a/cmd/bd/tag.go +++ b/cmd/bd/tag.go @@ -67,10 +67,10 @@ Examples: return HandleErrorRespectJSON("%s", err) } - if err := issueStore.AddLabel(ctx, result.ResolvedID, label, actor); err != nil { + if err := issueStore.AddLabel(ctx, result.ResolvedID, label, currentActor()); err != nil { return HandleErrorRespectJSON("adding label to %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "tag", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/unclaim.go b/cmd/bd/unclaim.go index f900e1750..3dd59a8fe 100644 --- a/cmd/bd/unclaim.go +++ b/cmd/bd/unclaim.go @@ -90,9 +90,9 @@ Examples: var unclaimErr error if conditional { - unclaimErr = issueStore.UnclaimIssueIfAssignee(ctx, fullID, actor, ifAssignee) + unclaimErr = issueStore.UnclaimIssueIfAssignee(ctx, fullID, currentActor(), ifAssignee) } else { - unclaimErr = issueStore.UnclaimIssue(ctx, fullID, actor, force) + unclaimErr = issueStore.UnclaimIssue(ctx, fullID, currentActor(), force) } if unclaimErr != nil { fmt.Fprintf(os.Stderr, "Error unclaiming %s: %v\n", fullID, unclaimErr) @@ -102,7 +102,7 @@ Examples: } if reason != "" { - if _, err := issueStore.AddIssueComment(ctx, fullID, actor, reason); err != nil { + if _, err := issueStore.AddIssueComment(ctx, fullID, currentActor(), reason); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to add reason comment: %v\n", err) } } diff --git a/cmd/bd/unclaim_proxied_server.go b/cmd/bd/unclaim_proxied_server.go index 85b56acc2..9269beb88 100644 --- a/cmd/bd/unclaim_proxied_server.go +++ b/cmd/bd/unclaim_proxied_server.go @@ -62,9 +62,9 @@ func runUnclaimProxiedServer(ctx context.Context, args []string, reason string, var uerr error if expectedAssignee != "" { - uerr = uw.IssueUseCase().UnclaimIfAssignee(ctx, fullID, actor, expectedAssignee) + uerr = uw.IssueUseCase().UnclaimIfAssignee(ctx, fullID, currentActor(), expectedAssignee) } else { - uerr = uw.IssueUseCase().Unclaim(ctx, fullID, actor, force) + uerr = uw.IssueUseCase().Unclaim(ctx, fullID, currentActor(), force) } if uerr != nil { r.errs = append(r.errs, fmt.Sprintf("Error unclaiming %s: %v", fullID, uerr)) @@ -72,7 +72,7 @@ func runUnclaimProxiedServer(ctx context.Context, args []string, reason string, } if reason != "" { - if _, cerr := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, actor, reason); cerr != nil { + if _, cerr := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, currentActor(), reason); cerr != nil { r.errs = append(r.errs, fmt.Sprintf("Warning: failed to add reason comment on %s: %v", fullID, cerr)) } } @@ -129,7 +129,7 @@ func runReclaimProxiedServer(ctx context.Context, olderThan time.Duration, filte } reclaimed, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) ([]types.ReclaimedLease, string, error) { - out, rerr := uw.IssueUseCase().ReclaimExpiredLeases(ctx, olderThan, filter, actor) + out, rerr := uw.IssueUseCase().ReclaimExpiredLeases(ctx, olderThan, filter, currentActor()) if rerr != nil { return nil, "", rerr } diff --git a/cmd/bd/undefer.go b/cmd/bd/undefer.go index 08e251c6a..b89907be8 100644 --- a/cmd/bd/undefer.go +++ b/cmd/bd/undefer.go @@ -89,7 +89,7 @@ Examples: updates["status"] = string(types.StatusOpen) } - if err := store.UpdateIssue(ctx, fullID, updates, actor); err != nil { + if err := store.UpdateIssue(ctx, fullID, updates, currentActor()); err != nil { fmt.Fprintf(os.Stderr, "Error undeferring %s: %v\n", fullID, err) continue } diff --git a/cmd/bd/update.go b/cmd/bd/update.go index 1510b8c78..09e8edb64 100644 --- a/cmd/bd/update.go +++ b/cmd/bd/update.go @@ -511,7 +511,7 @@ pointless).`, // the actor's own fresh claim. A policy refusal, so it exits 1, // not 13. if newAssignee, ok := updates["assignee"].(string); ok && ifAssignee == nil && !claimFlag && !ifRevisionAlreadyStale(issue, ifRevision) { - if err := validateIssueReassignable(id, issue, actor, newAssignee, + if err := validateIssueReassignable(id, issue, currentActor(), newAssignee, storeClaimPoolAliases(ctx, issueStore), forceFlag); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) recordFailure(id, err.Error()) @@ -552,7 +552,7 @@ pointless).`, // through: `--force -s closed` is now a legitimate way to ask for // the close-policy half alone. updateResult, updateErr := runCommandUpdateMutation(opsCtx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: result.ResolvedID, patch: patch, claim: claimFlag, @@ -602,13 +602,13 @@ pointless).`, } // Audit log key field changes (survives Dolt GC flatten) if patch.Status.Set { - audit.LogFieldChange(result.ResolvedID, "status", string(issue.Status), string(patch.Status.Value), actor, "") + audit.LogFieldChange(result.ResolvedID, "status", string(issue.Status), string(patch.Status.Value), currentActor(), "") } if patch.Assignee.Set { - audit.LogFieldChange(result.ResolvedID, "assignee", issue.Assignee, patch.Assignee.Value, actor, "") + audit.LogFieldChange(result.ResolvedID, "assignee", issue.Assignee, patch.Assignee.Value, currentActor(), "") } if patch.Priority.Set { - audit.LogFieldChange(result.ResolvedID, "priority", fmt.Sprintf("%d", issue.Priority), fmt.Sprintf("%d", patch.Priority.Value), actor, "") + audit.LogFieldChange(result.ResolvedID, "priority", fmt.Sprintf("%d", issue.Priority), fmt.Sprintf("%d", patch.Priority.Value), currentActor(), "") } // The operation's own post-state snapshot replaces the re-read. @@ -642,7 +642,7 @@ pointless).`, if s == nil { continue } - if err := commitPendingIfEmbedded(ctx, s, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, s, currentActor(), doltAutoCommitParams{ Command: "update", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/update_proxied_server.go b/cmd/bd/update_proxied_server.go index fee3e9225..5d74583b2 100644 --- a/cmd/bd/update_proxied_server.go +++ b/cmd/bd/update_proxied_server.go @@ -127,7 +127,7 @@ func applyUpdateProxiedOne(ctx context.Context, id string, in *updateInput) (*ty expectedStatus = &expected } result, err := runCommandUpdateMutation(ctx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: id, patch: patch, claim: in.claim, @@ -226,7 +226,7 @@ func proxiedUpdateTarget(ctx context.Context, id string, in *updateInput) (*type // mc-zndi7.74: also skipped when this pre-read is already stale against an // active --if-revision guard — see ifRevisionAlreadyStale's doc. if newAssignee, ok := in.fields["assignee"].(string); ok && in.ifAssignee == nil && !in.claim && !ifRevisionAlreadyStale(current, in.ifRevision) { - if err := validateIssueReassignable(id, current, actor, newAssignee, + if err := validateIssueReassignable(id, current, currentActor(), newAssignee, proxiedClaimPoolAliases(ctx), in.force); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) return nil, &updateIDFailure{ID: id, Error: err.Error()} diff --git a/cmd/bd/wisp.go b/cmd/bd/wisp.go index 329b1734f..a1b7ac27e 100644 --- a/cmd/bd/wisp.go +++ b/cmd/bd/wisp.go @@ -280,7 +280,7 @@ func runWispCreateCore(cmd *cobra.Command, args []string) error { result, err := spawnMoleculeWithOptions(ctx, store, subgraph, CloneOptions{ Vars: vars, - Actor: actor, + Actor: currentActor(), Ephemeral: true, Prefix: types.IDPrefixWisp, RootOnly: rootOnly, diff --git a/cmd/bd/wisp_proxied_server.go b/cmd/bd/wisp_proxied_server.go index edc727ea8..fd43df23b 100644 --- a/cmd/bd/wisp_proxied_server.go +++ b/cmd/bd/wisp_proxied_server.go @@ -92,7 +92,7 @@ func runWispCreateProxiedServer(ctx context.Context, in wispCreateInput) error { w := newUOWMolWriter(uw) spawnResult, err := cloneSubgraphInto(ctx, w, subgraph, CloneOptions{ Vars: vars, - Actor: actor, + Actor: currentActor(), Ephemeral: true, Prefix: types.IDPrefixWisp, RootOnly: in.rootOnly, @@ -189,7 +189,7 @@ func runWispGCProxiedServer(ctx context.Context, dryRun bool, ageThreshold time. IDs: ids, Cascade: false, UpdateTextReferences: true, - }, actor) + }, currentActor()) if err != nil { return domain.DeleteIssuesResult{}, "", err } @@ -320,7 +320,7 @@ func runWispPurgeClosedProxiedServer(ctx context.Context, dryRun, force bool, ex IDs: ids, Cascade: false, UpdateTextReferences: true, - }, actor) + }, currentActor()) if err != nil { return domain.DeleteIssuesResult{}, "", err } From a34de82ef0e74f1c443391c16c0042bd54db2117 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 19:13:34 +0000 Subject: [PATCH 003/151] fix(git): decline in-process discovery on any config line git might read differently Review follow-ups to the in-process repository discovery. - The repository config scanner was looser than git: a config starting with a UTF-8 BOM followed by `[core] worktree = /other` made git report /other while discovery returned the cwd repository, and `[ core ]` or `!!! = x` (which git refuses) were accepted. scanGitConfig (gitconfig_linux.go) now skips a leading BOM exactly as git does and accepts only a narrow grammar: blank/comment lines, `[section]` / `[section "sub"]` headers at column 0, and `key` / `key = value` lines inside a section, with values free of backslashes and unbalanced quotes. Any other line declines to git. - Walk from getcwd(2) (syscall.Getwd), as git does, instead of os.Getwd, which trusts $PWD when it names the same inode (bind-mount aliases). - Resolve relative gitfile and commondir targets without filepath.Join's lexical Clean, so "link/.." resolves physically like git's realpath. - A mount-boundary stop reports its own reason in the not-a-repository error. - Tests: every decline case (BOM + core.worktree, malformed headers and keys, keys before a section, escapes, continuations, bare repositories, a commondir inside .git, ...) now also checks that what startup derives equals git's answer; BOM and CRLF/comment configs are equivalence cases; the no-repository case must be answered in-process instead of skipping; a $PWD alias test pins the getcwd walk. - The exec-count test drops every GIT_* variable except the config-file selectors, so a runner exporting GIT_CONFIG_PARAMETERS & co. cannot make it fail spuriously. - Comment the two places that touch the raw actor before the deferral. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/bd/main.go | 8 ++ cmd/bd/startup_git_exec_embedded_test.go | 17 +++- internal/git/BUILD.bazel | 1 + internal/git/discover_linux.go | 108 +++++++++----------- internal/git/discover_linux_test.go | 119 ++++++++++++++++++++--- internal/git/gitconfig_linux.go | 109 +++++++++++++++++++++ internal/git/gitdir.go | 7 +- 7 files changed, 290 insertions(+), 79 deletions(-) create mode 100644 internal/git/gitconfig_linux.go diff --git a/cmd/bd/main.go b/cmd/bd/main.go index bdcb83bd1..901d461ee 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -840,6 +840,10 @@ func refreshBoundCommandConfig(cmd *cobra.Command) { readonlyMode = config.GetBool("readonly") } if !root.PersistentFlags().Changed("actor") { + // Raw write, not setActor: every caller runs in PersistentPreRunE + // BEFORE deferActorGitFallback, which then decides from this value + // whether the git user.name fallback is pending. A call after it would + // need setActor (and the deferral re-run) to keep that decision true. actor = resolveConfiguredActor() } if !root.PersistentFlags().Changed("dolt-auto-commit") { @@ -1302,6 +1306,10 @@ var rootCmd = &cobra.Command{ WasSet bool }{dbPath, true} } + // Reads and writes the raw global on purpose: this runs before + // deferActorGitFallback, so no fallback is pending yet and the value + // is only the flag/env/config actor. currentActor() here would resolve + // git user.name eagerly, which is what the deferral exists to avoid. if !cmd.Root().PersistentFlags().Changed("actor") && actor == "" { actor = resolveConfiguredActor() } else if cmd.Root().PersistentFlags().Changed("actor") { diff --git a/cmd/bd/startup_git_exec_embedded_test.go b/cmd/bd/startup_git_exec_embedded_test.go index 4073da792..a034bc0d7 100644 --- a/cmd/bd/startup_git_exec_embedded_test.go +++ b/cmd/bd/startup_git_exec_embedded_test.go @@ -41,9 +41,20 @@ func (r *gitExecRecorder) run(t *testing.T, bd, dir string, args ...string) (str } cmd := exec.Command(bd, args...) cmd.Dir = dir - env := envWithout(envWithout(bdEnv(dir), "BD_ACTOR"), "PATH") - for _, name := range []string{"GIT_DIR", "GIT_WORK_TREE", "GIT_COMMON_DIR", "GIT_CEILING_DIRECTORIES"} { - env = envWithout(env, name) + // Drop every GIT_* variable but the config-file selectors: any of + // internal/git's discovery overrides (GIT_DIR, GIT_WORK_TREE, + // GIT_COMMON_DIR, GIT_CEILING_DIRECTORIES, GIT_DISCOVERY_ACROSS_FILESYSTEM, + // GIT_OBJECT_DIRECTORY, GIT_CONFIG_PARAMETERS, GIT_CONFIG_COUNT with its + // GIT_CONFIG_KEY_n/VALUE_n, ...) exported by the runner — a `git -c` + // wrapper, a hook context — would rightly send bd back to git rev-parse. + var env []string + for _, kv := range envWithout(envWithout(bdEnv(dir), "BD_ACTOR"), "PATH") { + name, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(name, "GIT_") && name != "GIT_CONFIG_GLOBAL" && + name != "GIT_CONFIG_SYSTEM" && name != "GIT_CONFIG_NOSYSTEM" { + continue + } + env = append(env, kv) } // backup.enabled is set explicitly: left at its default, embedded-mode // auto-backup decides per command whether a git remote exists, which is diff --git a/internal/git/BUILD.bazel b/internal/git/BUILD.bazel index 3c5b1e305..e36e0ec55 100644 --- a/internal/git/BUILD.bazel +++ b/internal/git/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "discover_linux.go", "discover_other.go", + "gitconfig_linux.go", "gitdir.go", ], importpath = "github.com/steveyegge/beads/internal/git", diff --git a/internal/git/discover_linux.go b/internal/git/discover_linux.go index 7ffa7af33..11d0aec9e 100644 --- a/internal/git/discover_linux.go +++ b/internal/git/discover_linux.go @@ -3,8 +3,8 @@ package git import ( - "bufio" "bytes" + "errors" "os" "path/filepath" "strconv" @@ -38,13 +38,24 @@ func discoverGitInProcess() (revParseResult, bool) { return revParseResult{}, false } } - wd, err := os.Getwd() + wd, err := processCwd() if err != nil { return revParseResult{}, false } return discoverGitFrom(wd) } +// processCwd is getcwd(2), as Git uses. os.Getwd prefers $PWD whenever it +// names the same inode as ".", which under bind-mount aliases (same device +// and inode, different ancestors) would walk a different path than git. +func processCwd() (string, error) { + return syscall.Getwd() +} + +// errNotGitRepositoryMountBoundary is the in-process counterpart of Git's +// "not a git repository (or any parent up to mount point ...)". +var errNotGitRepositoryMountBoundary = errors.New("no .git found in the working directory or any parent up to the filesystem boundary") + // discoverGitFrom mirrors setup_git_directory_gently's discovery walk (Git // 2.x) from cwd. Git walks the physical directory (getcwd), checks each // directory's .git as a gitfile or a git directory, then the directory itself @@ -91,7 +102,7 @@ func discoverGitFrom(cwd string) (revParseResult, bool) { if parentStat.Dev != cwdStat.Dev { // Git stops at a mount point unless // GIT_DISCOVERY_ACROSS_FILESYSTEM is set (handled by the caller). - return revParseResult{notRepo: true}, true + return revParseResult{notRepo: true, notRepoErr: errNotGitRepositoryMountBoundary}, true } dir = parent } @@ -143,7 +154,10 @@ func discoveredGitFile(top, gitFile string) (revParseResult, bool) { return revParseResult{}, false } if !filepath.IsAbs(target) { - target = filepath.Join(top, target) + // Not filepath.Join: its lexical Clean would fold "link/.." before + // symlinks resolve, where Git's realpath resolves them physically + // (as EvalSymlinks does for ".." after a resolved component). + target = top + string(filepath.Separator) + target } gitDir, err := filepath.EvalSymlinks(target) if err != nil { @@ -156,7 +170,7 @@ func discoveredGitFile(top, gitFile string) (revParseResult, bool) { return revParseResult{}, false } if !filepath.IsAbs(common) { - common = filepath.Join(gitDir, common) + common = gitDir + string(filepath.Separator) + common // physical "..", as above } if commonDir, err = filepath.EvalSymlinks(common); err != nil { return revParseResult{}, false @@ -259,75 +273,43 @@ func submoduleWorktreeIs(gitDir, commonDir, worktree, top string) bool { // (read_repository_format), so global and system config cannot move the work // tree. Anything else that could — core.bare, extensions (worktreeConfig, // unknown ones Git rejects), a format version Git refuses, include -// directives, a repeated core.worktree, syntax this scanner does not model — -// sends discovery to git. +// directives, a repeated core.worktree, or any line scanGitConfig does not +// fully understand (where Git might honor or reject it) — sends discovery to +// git. func repoConfigAllowsDiscovery(commonDir string) (worktree string, ok bool) { - f, err := os.Open(filepath.Join(commonDir, "config")) // #nosec G304 -- the discovered repository config - if os.IsNotExist(err) { - return "", true - } - if err != nil { - return "", false - } - defer f.Close() - - section := "" - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" || line[0] == '#' || line[0] == ';' { - continue + ok = scanGitConfig(filepath.Join(commonDir, "config"), func(e gitConfigEntry) bool { + if strings.HasPrefix(e.section, "include") || e.section == "extensions" { + return false } - if strings.HasSuffix(line, "\\") { - return "", false // continuation lines are not modeled + if e.section != "core" || e.subsection != nil { + return true } - if line[0] == '[' { - end := strings.IndexByte(line, ']') - if end < 0 { - return "", false - } - header := strings.TrimSpace(line[1:end]) - if strings.ContainsAny(header, " \t\"") { - section = "" // a subsection: [remote "origin"], [branch "main"], ... - if name := strings.ToLower(strings.Fields(header)[0]); strings.HasPrefix(name, "include") { - return "", false - } - } else { - section = strings.ToLower(header) - } - if strings.HasPrefix(section, "include") || section == "extensions" { - return "", false - } - line = strings.TrimSpace(line[end+1:]) - if line == "" || line[0] == '#' || line[0] == ';' { - continue - } - } - if section != "core" { - continue - } - key, value, _ := strings.Cut(line, "=") - key = strings.ToLower(strings.TrimSpace(key)) - rawValue := value - value = strings.ToLower(strings.Trim(strings.TrimSpace(value), "\"")) - switch key { + switch e.key { case "worktree": - rawValue = strings.TrimSpace(rawValue) - if worktree != "" || rawValue == "" || strings.ContainsAny(rawValue, "\\\"#;") { - return "", false + if worktree != "" || !e.hasValue || e.value == "" || strings.ContainsAny(e.rawValue, "\"#;") { + return false } - worktree = rawValue + worktree = e.value case "bare": - if value != "false" && value != "no" && value != "off" && value != "0" { - return "", false + if !e.hasValue { + return false // a bare "bare" key means true + } + switch strings.ToLower(e.value) { + case "false", "no", "off", "0": + default: + return false } case "repositoryformatversion": - if v, err := strconv.Atoi(value); err != nil || v < 0 || v > 1 { - return "", false + if v, err := strconv.Atoi(e.value); err != nil || v < 0 || v > 1 { + return false } } + return true + }) + if !ok { + return "", false } - return worktree, scanner.Err() == nil + return worktree, true } // ownedByCurrentUser mirrors Git's is_path_owned_by_current_uid (lstat, owner diff --git a/internal/git/discover_linux_test.go b/internal/git/discover_linux_test.go index c956ba34d..e20a5ddb9 100644 --- a/internal/git/discover_linux_test.go +++ b/internal/git/discover_linux_test.go @@ -115,6 +115,43 @@ func newDiscoveryFixture(t *testing.T) discoveryFixture { return f } +// resolvedLikeStartup is what bd's startup derives for dir: the in-process +// answer when discovery gives one, else git's. +func resolvedLikeStartup(dir string, env []string) (gitContext, bool) { + if raw, ok := discoverGitFrom(dir); ok { + return gitContextFromRevParse(dir, raw), true + } + return loadGitContext(dir, env), false +} + +// assertSameAsGit checks the startup-derived context for dir against the git +// subprocess's: same failure, or the same paths. +func assertSameAsGit(t *testing.T, env []string, dir string) { + t.Helper() + got, _ := resolvedLikeStartup(dir, env) + want := loadGitContext(dir, env) + if (got.err != nil) != (want.err != nil) { + t.Fatalf("in %s: startup err %v, git err %v", dir, got.err, want.err) + } + if got.gitDirRaw != want.gitDirRaw || got.commonDir != want.commonDir || + got.repoRoot != want.repoRoot || got.isWorktree != want.isWorktree { + t.Fatalf("in %s: startup %+v, git %+v", dir, got, want) + } +} + +// repoWithRawConfig makes a repository and replaces its config file. +func repoWithRawConfig(t *testing.T, env []string, config string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "r") + runFixtureGit(t, env, t.TempDir(), "init", "-q", dir) + if err := os.WriteFile(filepath.Join(dir, ".git", "config"), []byte(config), 0o600); err != nil { + t.Fatal(err) + } + return dir +} + +const plainRepoConfig = "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n\tlogallrefupdates = true\n" + // TestDiscoverGitFromMatchesRevParse is the equivalence contract: wherever // in-process discovery answers, its answer is byte-for-byte what `git // rev-parse --git-dir --git-common-dir --show-toplevel` prints there. @@ -134,27 +171,29 @@ func TestDiscoverGitFromMatchesRevParse(t *testing.T) { {"nested repo subdir", filepath.Join(f.nested, "deep")}, {"symlinked path to repo top", filepath.Join(f.root, "link")}, {"symlinked path to repo subdir", filepath.Join(f.root, "link", "a", "b")}, + // No ancestor of a fresh temp directory is a repository, so this is + // answered in-process (by the walk reaching / or a mount boundary). {"no repository", filepath.Join(f.root, "norepo", "x")}, + {"config with a UTF-8 BOM", repoWithRawConfig(t, f.env, "\xef\xbb\xbf"+plainRepoConfig)}, + {"config with CRLF and comments", repoWithRawConfig(t, f.env, + "# leading comment\r\n[core] ; header comment\r\n\tbare = false # trailing\r\n[remote \"origin\"]\r\n\turl = \"/x y\"\r\n")}, } { t.Run(tc.name, func(t *testing.T) { got, ok := discoverGitFrom(tc.dir) if !ok { - if tc.name == "no repository" { - // An ancestor of the temp dir may be unusual (a bare - // repository, a foreign owner); declining is always safe. - t.Skip("in-process discovery declined above the fixture root") - } t.Fatalf("discoverGitFrom(%s) declined; want an in-process answer", tc.dir) } if want := gitRevParse(t, f.env, tc.dir); got != want { t.Fatalf("discoverGitFrom(%s) = %+v, git rev-parse = %+v", tc.dir, got, want) } + assertSameAsGit(t, f.env, tc.dir) }) } } // TestDiscoverGitFromDeclinesUnmodeledLayouts covers the layouts in-process -// discovery must hand to git rather than guess. +// discovery must hand to git rather than guess, and checks that what startup +// then derives (from git) is git's answer. func TestDiscoverGitFromDeclinesUnmodeledLayouts(t *testing.T) { f := newDiscoveryFixture(t) repoWithConfig := func(t *testing.T, args ...string) string { @@ -164,18 +203,27 @@ func TestDiscoverGitFromDeclinesUnmodeledLayouts(t *testing.T) { runFixtureGit(t, f.env, dir, append([]string{"config"}, args...)...) return dir } - invalidGitfile := filepath.Join(t.TempDir(), "badfile") - if err := os.MkdirAll(invalidGitfile, 0o750); err != nil { - t.Fatal(err) + mkdir := func(t *testing.T, parts ...string) string { + t.Helper() + dir := filepath.Join(parts...) + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + return dir } + invalidGitfile := mkdir(t, t.TempDir(), "badfile") if err := os.WriteFile(filepath.Join(invalidGitfile, ".git"), []byte("not a gitfile\n"), 0o600); err != nil { t.Fatal(err) } - symlinkedDotGit := filepath.Join(t.TempDir(), "symlinked") - if err := os.MkdirAll(symlinkedDotGit, 0o750); err != nil { + symlinkedDotGit := mkdir(t, t.TempDir(), "symlinked") + if err := os.Symlink(filepath.Join(f.main, ".git"), filepath.Join(symlinkedDotGit, ".git")); err != nil { t.Fatal(err) } - if err := os.Symlink(filepath.Join(f.main, ".git"), filepath.Join(symlinkedDotGit, ".git")); err != nil { + bare := filepath.Join(t.TempDir(), "bare.git") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", "--bare", bare) + bareSub := mkdir(t, bare, "sub") // below a bare repository: the walk meets HEAD+objects + commondirInDotGit := repoWithConfig(t, "core.filemode", "true") + if err := os.WriteFile(filepath.Join(commondirInDotGit, ".git", "commondir"), []byte(".\n"), 0o600); err != nil { t.Fatal(err) } for _, tc := range []struct{ name, dir string }{ @@ -183,7 +231,21 @@ func TestDiscoverGitFromDeclinesUnmodeledLayouts(t *testing.T) { {"core.bare", repoWithConfig(t, "core.bare", "true")}, {"extensions.worktreeConfig", repoWithConfig(t, "extensions.worktreeConfig", "true")}, {"include directive", repoWithConfig(t, "include.path", "/dev/null")}, + // Git skips a leading BOM and honors the section after it. + {"BOM then core.worktree", repoWithRawConfig(t, f.env, "\xef\xbb\xbf[core]\n\tworktree = "+t.TempDir()+"\n")}, + {"header with inner spaces", repoWithRawConfig(t, f.env, "[ core ]\n\tbare = false\n")}, + {"header with leading whitespace", repoWithRawConfig(t, f.env, " [core]\n\tbare = false\n")}, + {"malformed key line", repoWithRawConfig(t, f.env, plainRepoConfig+"!!! = x\n")}, + {"key before any section", repoWithRawConfig(t, f.env, "bare = false\n"+plainRepoConfig)}, + {"key on the header line", repoWithRawConfig(t, f.env, "[core] bare = false\n")}, + {"escaped value", repoWithRawConfig(t, f.env, plainRepoConfig+"[alias]\n\tx = \"a\\tb\"\n")}, + {"unterminated quote", repoWithRawConfig(t, f.env, plainRepoConfig+"[alias]\n\tx = \"a\n")}, + {"continuation line", repoWithRawConfig(t, f.env, plainRepoConfig+"[alias]\n\tx = a \\\n b\n")}, + {"dotted section", repoWithRawConfig(t, f.env, "[core.x]\n\ty = 1\n"+plainRepoConfig)}, {"inside the git directory", filepath.Join(f.main, ".git", "refs")}, + {"bare repository", bare}, + {"below a bare repository", bareSub}, + {"commondir inside .git", commondirInDotGit}, {"invalid gitfile", invalidGitfile}, {"symlinked .git", symlinkedDotGit}, } { @@ -191,10 +253,43 @@ func TestDiscoverGitFromDeclinesUnmodeledLayouts(t *testing.T) { if got, ok := discoverGitFrom(tc.dir); ok { t.Fatalf("discoverGitFrom(%s) = %+v; want it to defer to git", tc.dir, got) } + assertSameAsGit(t, f.env, tc.dir) }) } } +// TestDiscoverGitInProcessUsesPhysicalCwd: Git walks from getcwd(2); a $PWD +// naming the same directory through another path (a symlink here; a bind +// mount alias in containers) must not change the walk. +func TestDiscoverGitInProcessUsesPhysicalCwd(t *testing.T) { + f := newDiscoveryFixture(t) + physical := filepath.Join(f.main, "a", "b") + alias := filepath.Join(f.root, "link", "a", "b") + t.Chdir(physical) + t.Setenv("PWD", alias) + if wd, err := os.Getwd(); err != nil || wd != alias { + t.Fatalf("os.Getwd() = %q, %v; this test needs it to report the $PWD alias %q", wd, err, alias) + } + if wd, err := processCwd(); err != nil || wd != physical { + t.Fatalf("processCwd() = %q, %v; want getcwd's %q", wd, err, physical) + } + for _, k := range discoveryEnvOverrides { + if _, set := os.LookupEnv(k); set { + t.Setenv(k, "") + if err := os.Unsetenv(k); err != nil { + t.Fatal(err) + } + } + } + got, ok := discoverGitInProcess() + if !ok { + t.Fatal("discoverGitInProcess declined") + } + if want := gitRevParse(t, f.env, physical); got != want { + t.Fatalf("discoverGitInProcess() = %+v, git rev-parse = %+v", got, want) + } +} + // TestDiscoverGitInProcessDefersToGitEnvOverrides: GIT_DIR and friends // redefine discovery, so their presence (even empty) sends it to git. func TestDiscoverGitInProcessDefersToGitEnvOverrides(t *testing.T) { diff --git a/internal/git/gitconfig_linux.go b/internal/git/gitconfig_linux.go new file mode 100644 index 000000000..ea786b373 --- /dev/null +++ b/internal/git/gitconfig_linux.go @@ -0,0 +1,109 @@ +//go:build linux + +package git + +import ( + "bytes" + "os" + "regexp" + "strings" +) + +// gitConfigEntry is one key line of a git config file, as scanGitConfig +// understood it. +type gitConfigEntry struct { + section string // lower-cased section name + subsection *string // nil for [section]; case preserved + key string // lower-cased variable name + hasValue bool // false for a bare "key" line (boolean true) + rawValue string // value text after '=', as written, comment included + value string // rawValue without its comment, trimmed and unquoted +} + +var ( + gitConfigHeader = regexp.MustCompile(`^\[([A-Za-z0-9-]+)(?: "([^"\\]*)")?\][ \t]*(?:[#;].*)?$`) + gitConfigKeyLine = regexp.MustCompile(`^[ \t]*([A-Za-z][A-Za-z0-9-]*)[ \t]*(?:=(.*)|(?:[#;].*)?)$`) + gitConfigCommentLn = regexp.MustCompile(`^[ \t]*(?:[#;].*)?$`) +) + +// scanGitConfig reads a git config file with a deliberately narrow grammar +// and calls visit for each key line; it returns false (the caller then asks +// git) when the file is unreadable, a visit returns false, or ANY line falls +// outside that grammar — Git might honor such a line differently or refuse +// the file. A missing file is an empty config, as for Git. +// +// Understood: a UTF-8 BOM at the start of the file (Git skips it), CRLF line +// endings, blank and comment lines, "[section]" and `[section "sub"]` +// headers at the start of a line (optionally followed by a comment), and +// "key", "key = value" lines inside a section. Values containing a backslash +// or an odd number of double quotes (escapes, continuations, unterminated +// strings) are not understood. +func scanGitConfig(path string, visit func(gitConfigEntry) bool) bool { + data, err := os.ReadFile(path) // #nosec G304 -- a git config file Git itself would read + if os.IsNotExist(err) { + return true + } + if err != nil || bytes.IndexByte(data, 0) >= 0 { + return false + } + text := strings.TrimPrefix(string(data), "\xef\xbb\xbf") + section, haveSection := "", false + var subsection *string + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSuffix(line, "\r") + if strings.ContainsRune(line, '\r') { + return false + } + if gitConfigCommentLn.MatchString(line) { + continue + } + if m := gitConfigHeader.FindStringSubmatch(line); m != nil { + section, haveSection = strings.ToLower(m[1]), true + subsection = nil + if strings.Contains(line[:strings.IndexByte(line, ']')], `"`) { + sub := m[2] + subsection = &sub + } + continue + } + m := gitConfigKeyLine.FindStringSubmatchIndex(line) + if m == nil || !haveSection { + return false + } + e := gitConfigEntry{section: section, subsection: subsection, key: strings.ToLower(line[m[2]:m[3]])} + if m[4] >= 0 { + e.hasValue = true + e.rawValue = line[m[4]:m[5]] + value, ok := parseGitConfigValue(e.rawValue) + if !ok { + return false + } + e.value = value + } + if !visit(e) { + return false + } + } + return true +} + +// parseGitConfigValue strips a trailing comment, surrounding whitespace and +// balanced double quotes from a value it fully understands. +func parseGitConfigValue(raw string) (string, bool) { + if strings.ContainsRune(raw, '\\') || strings.Count(raw, `"`)%2 != 0 { + return "", false + } + var b strings.Builder + inQuote := false + for _, c := range raw { + switch { + case c == '"': + inQuote = !inQuote + continue + case !inQuote && (c == '#' || c == ';'): + return strings.TrimSpace(b.String()), true + } + b.WriteRune(c) + } + return strings.TrimSpace(b.String()), true +} diff --git a/internal/git/gitdir.go b/internal/git/gitdir.go index bb53eb63a..133ed8203 100644 --- a/internal/git/gitdir.go +++ b/internal/git/gitdir.go @@ -111,6 +111,7 @@ func initGitContext() { type revParseResult struct { gitDir, commonDir, topLevel string notRepo bool + notRepoErr error // why, when notRepo; nil means errNotGitRepository } // errNotGitRepository stands in for git's exit status when in-process @@ -154,7 +155,11 @@ func loadGitContext(workDir string, env []string) gitContext { func gitContextFromRevParse(workDir string, raw revParseResult) gitContext { var ctx gitContext if raw.notRepo { - ctx.err = fmt.Errorf("not a git repository: %w", errNotGitRepository) + reason := raw.notRepoErr + if reason == nil { + reason = errNotGitRepository + } + ctx.err = fmt.Errorf("not a git repository: %w", reason) return ctx } ctx.gitDirRaw = raw.gitDir From 418c89a001414ff551a973610a7e3e36dba6705a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 19:18:44 +0000 Subject: [PATCH 004/151] perf(backup): decide embedded auto-backup's git-remote probe without git In an embedded workspace whose backup.enabled is unset, maybeAutoBackup asks isBackupAutoEnabled -> primeHasGitRemote after every command, reads included. That built a RepoContext (`git -C .beads rev-parse --git-common-dir`) and ran `git remote`: two git processes per read even after the startup calls were gone. The throttle cannot short-circuit it: without a remote no backup ever runs, so there is no backup state to throttle on. Both probes are now answered in-process when provable, and the subprocess remains the fallback, so the decision and every warning/notice downstream of it are unchanged: - internal/beads getGitCommonDirForPath uses git.CommonDirInProcess (the discovery walk from the given directory, git's spelling). - primeHasGitRemote uses git.HasRemoteInProcess in the same directory primeGitCmd would run git in (primeGitDir, factored out). It answers only when every config file `git remote` reads -- system (GIT_CONFIG_SYSTEM, or /etc/gitconfig and /etc/gitconfig of the git on PATH), global (GIT_CONFIG_GLOBAL, or XDG and ~/.gitconfig) and the repository's own -- is absent or within scanGitConfig's grammar and include-free, no GIT_CONFIG_PARAMETERS/COUNT/GIT_CONFIG or discovery override is set, and no legacy .git/remotes or .git/branches entries exist. A remote seen only in a system-config candidate declines. Remote names outside [A-Za-z0-9._-] decline. Tests compare HasRemoteInProcess with `git remote` (no remote, local, worktree, submodule, remote.pushDefault only, global remote, no repo) and pin the declines (global include, malformed global, command-line config); CommonDirInProcess is compared with `git -C dir rev-parse --git-common-dir`. TestEmbeddedReadOnlyCommandsSpawnNoGit now runs with backup.enabled at its default, with and without a git remote, and still expects zero git processes for list/show. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 +- cmd/bd/prime.go | 29 ++- cmd/bd/startup_git_exec_embedded_test.go | 70 +++++--- internal/beads/context.go | 20 ++- internal/git/discover_linux.go | 214 ++++++++++++++++++++++- internal/git/discover_linux_test.go | 129 ++++++++++++++ internal/git/discover_other.go | 10 ++ 7 files changed, 439 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98882f789..9831b832c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,7 +124,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ownership subject to `safe.directory`). The `git config user.name` actor fallback is resolved only when a command actually needs an actor, so read-only commands such as `bd list` and `bd show` start no git process - for it. The actor's value and priority order are unchanged. + for it. The actor's value and priority order are unchanged. In an embedded + workspace with `backup.enabled` unset, the per-command auto-backup check + ("is there a git remote?") is likewise answered from the git config files + when they are plain (no includes, no command-line config), instead of + running `git -C .beads rev-parse --git-common-dir` and `git remote`. - `bd create --graph` now stores a plan whose path from a node's parent to the node runs through a `waits-for` edge; the graph-only preflight that diff --git a/cmd/bd/prime.go b/cmd/bd/prime.go index adec80396..f314fcb4a 100644 --- a/cmd/bd/prime.go +++ b/cmd/bd/prime.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads" internalbeads "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/memoryops" ) @@ -303,10 +304,16 @@ func primeWorkspaceDir() string { // failure falls back to a cwd probe instead of propagating — so this no // longer returns an error. If a future caller needs one, reintroduce it then. func primeGitCmd(ctx context.Context, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Dir = primeGitDir() + return cmd +} + +// primeGitDir is the directory primeGitCmd runs git in ("" for the process +// cwd): the -C target, else the cwd repository root from the RepoContext. +func primeGitDir() string { if ws := primeWorkspaceDir(); ws != "" { - cmd := exec.CommandContext(ctx, "git", args...) - cmd.Dir = ws - return cmd + return ws } rc, err := internalbeads.GetRepoContext() if err != nil { @@ -317,9 +324,9 @@ func primeGitCmd(ctx context.Context, args ...string) *exec.Cmd { // workspace, so probe it directly instead of giving up. The SEC-003 // boundary on BEADS_DIR stays enforced by the callers that consume // BEADS_DIR itself; these probes only ask git about its own workspace. - return exec.CommandContext(ctx, "git", args...) + return "" } - return rc.GitCmdCWD(ctx, args...) + return rc.CWDRepoRoot // GitCmdCWD's directory; "" leaves the process cwd } // outputHookJSON wraps content in the SessionStart hook JSON envelope shared @@ -422,8 +429,18 @@ var primeAgentProfile = func() config.AgentProfile { // `git remote` in the workspace succeeds. primeGitCmd supplies that fallback, // which keeps the -C behavior above intact. The SEC-003 boundary on BEADS_DIR // remains enforced elsewhere. +// +// The probe runs on every command in an embedded workspace whose +// backup.enabled is unset (isBackupAutoEnabled), so it is answered from the +// config files when git.HasRemoteInProcess can prove the answer, and by +// `git remote` otherwise. var primeHasGitRemote = func() bool { - cmd := primeGitCmd(context.Background(), "remote") + dir := primeGitDir() + if has, ok := git.HasRemoteInProcess(dir); ok { + return has + } + cmd := exec.CommandContext(context.Background(), "git", "remote") + cmd.Dir = dir out, err := cmd.Output() if err != nil { return false diff --git a/cmd/bd/startup_git_exec_embedded_test.go b/cmd/bd/startup_git_exec_embedded_test.go index a034bc0d7..a74194207 100644 --- a/cmd/bd/startup_git_exec_embedded_test.go +++ b/cmd/bd/startup_git_exec_embedded_test.go @@ -56,11 +56,20 @@ func (r *gitExecRecorder) run(t *testing.T, bd, dir string, args ...string) (str } env = append(env, kv) } - // backup.enabled is set explicitly: left at its default, embedded-mode - // auto-backup decides per command whether a git remote exists, which is - // a post-command probe of its own and not the startup path pinned here. + // backup.enabled stays at its default, so embedded auto-backup asks + // whether a git remote exists after every command (isBackupAutoEnabled). + // Global and system git config are pinned (empty, none) so the answer + // does not depend on the host's ~/.gitconfig — an include there would + // rightly send that probe to `git remote`. env = envWithout(envWithout(env, "BD_BACKUP_ENABLED"), "BEADS_BACKUP_ENABLED") - cmd.Env = append(env, "BD_BACKUP_ENABLED=false", + for _, name := range []string{"GIT_CONFIG_GLOBAL", "GIT_CONFIG_SYSTEM", "GIT_CONFIG_NOSYSTEM"} { + env = envWithout(env, name) + } + emptyGlobal := filepath.Join(r.shimDir, "empty.gitconfig") + if err := os.WriteFile(emptyGlobal, nil, 0o600); err != nil { + t.Fatal(err) + } + cmd.Env = append(env, "GIT_CONFIG_GLOBAL="+emptyGlobal, "GIT_CONFIG_NOSYSTEM=1", "PATH="+r.shimDir+string(os.PathListSeparator)+os.Getenv("PATH")) out, err := cmd.Output() if err != nil { @@ -83,11 +92,13 @@ func (r *gitExecRecorder) run(t *testing.T, bd, dir string, args ...string) (str return string(out), calls } -// TestEmbeddedReadOnlyCommandsSpawnNoGit pins the startup cost tools that call -// bd by the hundred depend on: a read-only command resolves the repository -// in-process and never needs the actor, so it runs no git at all — from the -// main checkout and from a linked worktree sharing its database. A write still -// resolves the actor from git config user.name, lazily and to the same value. +// TestEmbeddedReadOnlyCommandsSpawnNoGit pins the per-call cost tools that +// call bd by the hundred depend on: a read-only command resolves the +// repository in-process, never needs the actor, and decides embedded +// auto-backup (default backup.enabled, with and without a git remote) without +// git, so it runs no git at all — from the main checkout and from a linked +// worktree sharing its database. A write still resolves the actor from git +// config user.name, lazily and to the same value. func TestEmbeddedReadOnlyCommandsSpawnNoGit(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") @@ -114,24 +125,37 @@ func TestEmbeddedReadOnlyCommandsSpawnNoGit(t *testing.T) { } rec := newGitExecRecorder(t) - for _, location := range []struct{ name, dir string }{ - {"main checkout", dir}, - {"linked worktree", worktree}, - } { - for _, args := range [][]string{ - {"list", "--json"}, - {"show", issue.ID, "--json"}, + reads := func(phase string) { + t.Helper() + for _, location := range []struct{ name, dir string }{ + {"main checkout", dir}, + {"linked worktree", worktree}, } { - out, calls := rec.run(t, bd, location.dir, args...) - if !strings.Contains(out, issue.ID) { - t.Fatalf("%s: bd %s did not find %s:\n%s", location.name, args[0], issue.ID, out) - } - if len(calls) != 0 { - t.Errorf("%s: bd %s spawned %d git process(es), want 0:\n%s", - location.name, args[0], len(calls), strings.Join(calls, "\n")) + for _, args := range [][]string{ + {"list", "--json"}, + {"show", issue.ID, "--json"}, + } { + out, calls := rec.run(t, bd, location.dir, args...) + if !strings.Contains(out, issue.ID) { + t.Fatalf("%s, %s: bd %s did not find %s:\n%s", phase, location.name, args[0], issue.ID, out) + } + if len(calls) != 0 { + t.Errorf("%s, %s: bd %s spawned %d git process(es), want 0:\n%s", + phase, location.name, args[0], len(calls), strings.Join(calls, "\n")) + } } } } + // No git remote: auto-backup stays off, decided without running git. + reads("no git remote") + // With a remote, auto-backup turns on (still decided in-process); the + // first read may run the backup itself, which needs no git either. + addRemote := gitCommand("remote", "add", "origin", "https://example.invalid/gitcalls.git") + addRemote.Dir, addRemote.Env = dir, bdEnv(dir) + if out, err := addRemote.CombinedOutput(); err != nil { + t.Fatalf("git remote add: %v\n%s", err, out) + } + reads("git remote configured") // Positive control for the shim, and the deferred actor's value: a write // asks git for user.name exactly once and records it as before. diff --git a/internal/beads/context.go b/internal/beads/context.go index 63c63394c..7db3b0ee8 100644 --- a/internal/beads/context.go +++ b/internal/beads/context.go @@ -314,13 +314,23 @@ func isExternalBeadsDir(beadsDir string) (bool, error) { // getGitCommonDirForPath returns the shared git directory for a path. // For worktrees, this returns the shared git directory (common to all worktrees). +// +// Every startup that builds a RepoContext asks this about the beads +// directory, so the answer comes from git.CommonDirInProcess when it can, +// with `git -C path rev-parse --git-common-dir` as the fallback. func getGitCommonDirForPath(path string) (string, error) { - cmd := exec.Command("git", "-C", path, "rev-parse", "--git-common-dir") - output, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("failed to get git common dir for %s: %w", path, err) + result, isRepo, ok := git.CommonDirInProcess(path) + switch { + case ok && !isRepo: + return "", fmt.Errorf("failed to get git common dir for %s: not a git repository", path) + case !ok: + cmd := exec.Command("git", "-C", path, "rev-parse", "--git-common-dir") + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to get git common dir for %s: %w", path, err) + } + result = strings.TrimSpace(string(output)) } - result := strings.TrimSpace(string(output)) if !filepath.IsAbs(result) { absPath, err := filepath.Abs(path) diff --git a/internal/git/discover_linux.go b/internal/git/discover_linux.go index 11d0aec9e..4f88fd903 100644 --- a/internal/git/discover_linux.go +++ b/internal/git/discover_linux.go @@ -6,7 +6,9 @@ import ( "bytes" "errors" "os" + "os/exec" "path/filepath" + "regexp" "strconv" "strings" "syscall" @@ -33,10 +35,8 @@ var discoveryEnvOverrides = []string{ // is false whenever the layout is outside what it reproduces exactly; the // caller then runs git. func discoverGitInProcess() (revParseResult, bool) { - for _, name := range discoveryEnvOverrides { - if _, set := os.LookupEnv(name); set { - return revParseResult{}, false - } + if discoveryEnvOverridden() { + return revParseResult{}, false } wd, err := processCwd() if err != nil { @@ -351,3 +351,209 @@ func isDir(path string) bool { info, err := os.Stat(path) return err == nil && info.IsDir() } + +// CommonDirInProcess answers `git -C dir rev-parse --git-common-dir` without +// running git, in git's own spelling (relative to dir's physical location +// when git prints it relative). isRepo is false where git would fail with +// "not a git repository". ok is false whenever discoverGitInProcess would +// decline; the caller then runs git. +func CommonDirInProcess(dir string) (commonDir string, isRepo, ok bool) { + if discoveryEnvOverridden() { + return "", false, false + } + abs, err := filepath.Abs(dir) + if err != nil { + return "", false, false + } + raw, ok := discoverGitFrom(abs) + if !ok { + return "", false, false + } + if raw.notRepo { + return "", false, true + } + return raw.commonDir, true, true +} + +// HasRemoteInProcess answers "does `git remote` in dir print anything" (dir +// "" is the process working directory) without running git, or ok=false. +// +// `git remote` lists every remote that any config scope defines a +// remote..* key for, and fails outside a repository. The answer is +// given only when every file git would read — system, global (XDG and +// ~/.gitconfig, or GIT_CONFIG_GLOBAL), and the repository's own config — is +// absent or within scanGitConfig's grammar and free of include directives, +// the repository is one discoverGitFrom answers for, and no environment +// variable adds config or redirects discovery. The system file's location is +// compiled into git; the candidates are /etc/gitconfig and /etc/gitconfig +// for the git found on PATH (as invoked and with symlinks resolved), and a +// remote seen only in a system candidate declines rather than guess which one +// git reads. +func HasRemoteInProcess(dir string) (has, ok bool) { + if discoveryEnvOverridden() { + return false, false + } + if _, set := os.LookupEnv("GIT_CONFIG"); set { + return false, false + } + if dir == "" { + wd, err := processCwd() + if err != nil { + return false, false + } + dir = wd + } + abs, err := filepath.Abs(dir) + if err != nil { + return false, false + } + physical, err := filepath.EvalSymlinks(abs) + if err != nil { + return false, false + } + raw, ok := discoverGitFrom(physical) + if !ok { + return false, false + } + if raw.notRepo { + return false, true // `git remote` exits 128 outside a repository + } + commonDir := raw.commonDir + if !filepath.IsAbs(commonDir) { + commonDir = filepath.Join(physical, commonDir) + } + // Legacy remote definitions (.git/remotes/*, .git/branches/*) are not + // listed by `git remote`, but are not modeled either: decline if present. + for _, legacy := range []string{"remotes", "branches"} { + if entries, err := os.ReadDir(filepath.Join(commonDir, legacy)); err == nil && len(entries) > 0 { + return false, false + } else if err != nil && !os.IsNotExist(err) { + return false, false + } + } + + system, ok := systemConfigCandidates() + if !ok { + return false, false + } + global, ok := globalConfigFiles() + if !ok { + return false, false + } + inSystem, inOther := false, false + for _, scope := range []struct { + files []string + system bool + }{{system, true}, {global, false}, {[]string{filepath.Join(commonDir, "config")}, false}} { + for _, file := range scope.files { + found, ok := configDefinesRemote(file) + if !ok { + return false, false + } + if found && scope.system { + inSystem = true + } else if found { + inOther = true + } + } + } + switch { + case inOther: + return true, true + case inSystem: + return false, false + default: + return false, true + } +} + +// remoteName is the remote-name shape answered in-process; git ignores names +// starting with "/" and accepts many others this does not model. +var remoteName = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) + +// configDefinesRemote reports whether file defines a remote..* key. +func configDefinesRemote(file string) (found, ok bool) { + ok = scanGitConfig(file, func(e gitConfigEntry) bool { + if strings.HasPrefix(e.section, "include") { + return false + } + if e.section != "remote" || e.subsection == nil { + return true // remote.pushDefault and friends define no remote + } + if !remoteName.MatchString(*e.subsection) { + return false + } + found = true + return true + }) + return found, ok +} + +// globalConfigFiles mirrors git_global_config. +func globalConfigFiles() ([]string, bool) { + if global, set := os.LookupEnv("GIT_CONFIG_GLOBAL"); set { + if global == "" || !filepath.IsAbs(global) { + return nil, false + } + return []string{global}, true + } + home := os.Getenv("HOME") + if home == "" || !filepath.IsAbs(home) { + return nil, false + } + xdg := filepath.Join(home, ".config", "git", "config") + if x := os.Getenv("XDG_CONFIG_HOME"); x != "" { + if !filepath.IsAbs(x) { + return nil, false + } + xdg = filepath.Join(x, "git", "config") + } + return []string{xdg, filepath.Join(home, ".gitconfig")}, true +} + +// systemConfigCandidates mirrors git_system_config, with the compiled-in +// ETC_GITCONFIG approximated as described on HasRemoteInProcess. +func systemConfigCandidates() ([]string, bool) { + if v, set := os.LookupEnv("GIT_CONFIG_NOSYSTEM"); set { + switch strings.ToLower(v) { + case "1", "true", "yes", "on": + return nil, true + case "", "0", "false", "no", "off": + default: + return nil, false + } + } + if system, set := os.LookupEnv("GIT_CONFIG_SYSTEM"); set { + if system == "" || !filepath.IsAbs(system) { + return nil, false + } + return []string{system}, true + } + bin, err := exec.LookPath("git") + if err != nil || !filepath.IsAbs(bin) { + return nil, false + } + candidates := []string{"/etc/gitconfig"} + paths := []string{bin} + if resolved, err := filepath.EvalSymlinks(bin); err == nil { + paths = append(paths, resolved) + } + for _, p := range paths { + if filepath.Base(filepath.Dir(p)) != "bin" { + return nil, false + } + if prefix := filepath.Dir(filepath.Dir(p)); prefix != "/usr" { + candidates = append(candidates, filepath.Join(prefix, "etc", "gitconfig")) + } + } + return candidates, true +} + +func discoveryEnvOverridden() bool { + for _, name := range discoveryEnvOverrides { + if _, set := os.LookupEnv(name); set { + return true + } + } + return false +} diff --git a/internal/git/discover_linux_test.go b/internal/git/discover_linux_test.go index e20a5ddb9..04d111f25 100644 --- a/internal/git/discover_linux_test.go +++ b/internal/git/discover_linux_test.go @@ -340,3 +340,132 @@ func TestGitContextInProcessMatchesGitSubprocess(t *testing.T) { }) } } + +// isolateGitConfigEnv points git and the in-process readers at the same, +// controlled config files: no system config, an empty global file. +func isolateGitConfigEnv(t *testing.T) (global string) { + t.Helper() + for _, kv := range os.Environ() { + name, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(name, "GIT_") { + t.Setenv(name, "") + if err := os.Unsetenv(name); err != nil { + t.Fatal(err) + } + } + } + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "xdg")) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + global = filepath.Join(home, "global.gitconfig") + if err := os.WriteFile(global, nil, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("GIT_CONFIG_GLOBAL", global) + return global +} + +func gitRemoteListsAny(t *testing.T, dir string) bool { + t.Helper() + cmd := exec.Command("git", "remote") + cmd.Dir = dir + out, err := cmd.Output() + return err == nil && strings.TrimSpace(string(out)) != "" +} + +// TestHasRemoteInProcessMatchesGitRemote: wherever HasRemoteInProcess +// answers, it is what `git remote` (the auto-backup probe) reports. +func TestHasRemoteInProcessMatchesGitRemote(t *testing.T) { + f := newDiscoveryFixture(t) + global := isolateGitConfigEnv(t) + withRemote := filepath.Join(t.TempDir(), "with-remote") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", withRemote) + runFixtureGit(t, f.env, withRemote, "remote", "add", "origin", "https://example.invalid/r.git") + pushDefaultOnly := filepath.Join(t.TempDir(), "push-default") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", pushDefaultOnly) + runFixtureGit(t, f.env, pushDefaultOnly, "config", "remote.pushDefault", "origin") + runFixtureGit(t, f.env, f.main, "remote", "add", "up", "https://example.invalid/m.git") + + setGlobal := func(t *testing.T, content string) { + t.Helper() + if err := os.WriteFile(global, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.WriteFile(global, nil, 0o600) }) + } + for _, tc := range []struct { + name, dir, global string + wantDecline bool + }{ + {name: "no remote", dir: f.nested}, + {name: "local remote", dir: withRemote}, + {name: "remote.pushDefault only", dir: pushDefaultOnly}, + {name: "linked worktree sees the common config", dir: filepath.Join(f.worktree, "w")}, + {name: "submodule (origin in its own config)"}, + {name: "no repository", dir: filepath.Join(f.root, "norepo", "x")}, + {name: "remote in global config", dir: f.nested, global: "[remote \"g\"]\n\turl = /g\n"}, + {name: "include in global config", dir: f.nested, global: "[include]\n\tpath = /dev/null\n", wantDecline: true}, + {name: "malformed global config", dir: f.nested, global: "[ user ]\n", wantDecline: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := tc.dir + if dir == "" { + dir = f.submodule + } + if tc.global != "" { + setGlobal(t, tc.global) + } + has, ok := HasRemoteInProcess(dir) + if ok == tc.wantDecline { + t.Fatalf("HasRemoteInProcess(%s) ok=%v, want ok=%v", dir, ok, !tc.wantDecline) + } + if want := gitRemoteListsAny(t, dir); ok && has != want { + t.Fatalf("HasRemoteInProcess(%s) = %v, git remote lists any = %v", dir, has, want) + } + }) + } + t.Run("GIT_CONFIG_PARAMETERS declines", func(t *testing.T) { + t.Setenv("GIT_CONFIG_PARAMETERS", "'remote.x.url'='/x'") + if _, ok := HasRemoteInProcess(f.nested); ok { + t.Fatal("HasRemoteInProcess answered despite command-line config") + } + }) + t.Run("process cwd", func(t *testing.T) { + t.Chdir(withRemote) + if has, ok := HasRemoteInProcess(""); !ok || !has { + t.Fatalf("HasRemoteInProcess(\"\") = %v, %v; want true, true", has, ok) + } + }) +} + +// TestCommonDirInProcessMatchesGit: the auto-backup RepoContext asks +// `git -C <.beads> rev-parse --git-common-dir`; the in-process answer has +// git's spelling. +func TestCommonDirInProcessMatchesGit(t *testing.T) { + f := newDiscoveryFixture(t) + isolateGitConfigEnv(t) + beadsDir := filepath.Join(f.main, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + for _, dir := range []string{ + f.main, beadsDir, filepath.Join(f.main, "a", "b"), f.worktree, f.submodule, + filepath.Join(f.root, "link", "a"), filepath.Join(f.root, "norepo", "x"), + } { + t.Run(strings.TrimPrefix(dir, f.root), func(t *testing.T) { + got, isRepo, ok := CommonDirInProcess(dir) + if !ok { + t.Fatalf("CommonDirInProcess(%s) declined", dir) + } + cmd := exec.Command("git", "-C", dir, "rev-parse", "--git-common-dir") + out, err := cmd.Output() + if (err == nil) != isRepo { + t.Fatalf("CommonDirInProcess(%s) isRepo=%v, git err=%v", dir, isRepo, err) + } + if want := strings.TrimSpace(string(out)); isRepo && got != want { + t.Fatalf("CommonDirInProcess(%s) = %q, git = %q", dir, got, want) + } + }) + } +} diff --git a/internal/git/discover_other.go b/internal/git/discover_other.go index 4ba6e1083..231616c9b 100644 --- a/internal/git/discover_other.go +++ b/internal/git/discover_other.go @@ -8,3 +8,13 @@ package git func discoverGitInProcess() (revParseResult, bool) { return revParseResult{}, false } + +// CommonDirInProcess is linux-only; see discoverGitInProcess. +func CommonDirInProcess(string) (commonDir string, isRepo, ok bool) { + return "", false, false +} + +// HasRemoteInProcess is linux-only; see discoverGitInProcess. +func HasRemoteInProcess(string) (has, ok bool) { + return false, false +} From 5126de8c1a02d01db2481e4fd5e78dd41010e25f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 20:08:24 +0000 Subject: [PATCH 005/151] perf(bd): no git subprocesses on read-only commands (in-process repo discovery, lazy actor, backup probe) (#7400) * perf(git): locate the repository in-process instead of running git rev-parse Every bd invocation resolved the process-wide git context (beads directory discovery reads it) with `git rev-parse --git-dir --git-common-dir --show-toplevel`: one git subprocess per bd call before any work. Tools that call bd hundreds of times (gascity's doctor issues ~250 bd calls per run) paid it on every call. On Linux, initGitContext now first runs discoverGitInProcess, which mirrors Git's discovery walk (physical cwd, .git as directory or gitfile, bare-dir check, filesystem-boundary stop) and produces byte-for-byte what rev-parse prints for ordinary repositories (top and subdirectories), linked worktrees (absolute or relative gitfile, symlinked targets), submodules (core.worktree pointing back at the gitfile's directory), nested repositories and no repository. Both answers feed one shared normalization (gitContextFromRevParse), so downstream values are unchanged. It declines, and git runs exactly as before, whenever it cannot prove the answer: GIT_DIR / GIT_WORK_TREE / GIT_COMMON_DIR / GIT_CEILING_DIRECTORIES / GIT_DISCOVERY_ACROSS_FILESYSTEM / GIT_OBJECT_DIRECTORY / GIT_CONFIG_PARAMETERS / GIT_CONFIG_COUNT in the environment; a symlinked .git, an invalid gitfile or git directory, a cwd inside a git directory; repository config with core.worktree (other than the submodule shape), core.bare, extensions, include directives, a format version above 1 or unmodeled syntax; any path not owned by the effective uid (safe.directory is git's to evaluate), or root under sudo. darwin and windows keep the subprocess (case folding and ownership semantics). Tests compare discoverGitFrom with real `git rev-parse` output across those layouts, pin every decline case, and check the resulting cached context equals the subprocess-derived one. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * perf(cli): resolve the git user.name actor fallback only when needed PersistentPreRunE assigned `actor = getActorWithGit()` for every command, which, with no --actor / BEADS_ACTOR / BD_ACTOR / config actor, runs `git config user.name`: the second git subprocess of every bd call, paid by read-only commands that never consult the actor. The pre-run now calls deferActorGitFallback, which marks an empty actor as pending; currentActor (and getActorWithGit, getActor) resolve it on first use with the unchanged priority order and store it in the global and the CommandContext copy, as the eager assignment did. Command code reads the actor through currentActor() instead of the raw global (a mechanical rewrite of every read site the compiler reported; writes, the flag binding and the pre-run config binding keep the raw variable). setActor cancels a pending fallback. The telemetry span's bd.actor attribute is still set, but only for a recording span, so a noop span does not force the lookup. With the in-process repository discovery from the previous commit, a read-only `bd list` / `bd show` now starts no git process at startup, in the main checkout and in a linked worktree; a write asks git for user.name once (bd create previously asked once, now still once) and records the same value. Measured on 50 sequential `bd list` calls: 100 -> 0 git execs (embedded and proxied-server workspaces, main checkout and worktree). TestEmbeddedReadOnlyCommandsSpawnNoGit counts git execs through a PATH shim; TestDeferredActorGitFallback pins the deferral semantics. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * fix(git): decline in-process discovery on any config line git might read differently Review follow-ups to the in-process repository discovery. - The repository config scanner was looser than git: a config starting with a UTF-8 BOM followed by `[core] worktree = /other` made git report /other while discovery returned the cwd repository, and `[ core ]` or `!!! = x` (which git refuses) were accepted. scanGitConfig (gitconfig_linux.go) now skips a leading BOM exactly as git does and accepts only a narrow grammar: blank/comment lines, `[section]` / `[section "sub"]` headers at column 0, and `key` / `key = value` lines inside a section, with values free of backslashes and unbalanced quotes. Any other line declines to git. - Walk from getcwd(2) (syscall.Getwd), as git does, instead of os.Getwd, which trusts $PWD when it names the same inode (bind-mount aliases). - Resolve relative gitfile and commondir targets without filepath.Join's lexical Clean, so "link/.." resolves physically like git's realpath. - A mount-boundary stop reports its own reason in the not-a-repository error. - Tests: every decline case (BOM + core.worktree, malformed headers and keys, keys before a section, escapes, continuations, bare repositories, a commondir inside .git, ...) now also checks that what startup derives equals git's answer; BOM and CRLF/comment configs are equivalence cases; the no-repository case must be answered in-process instead of skipping; a $PWD alias test pins the getcwd walk. - The exec-count test drops every GIT_* variable except the config-file selectors, so a runner exporting GIT_CONFIG_PARAMETERS & co. cannot make it fail spuriously. - Comment the two places that touch the raw actor before the deferral. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * perf(backup): decide embedded auto-backup's git-remote probe without git In an embedded workspace whose backup.enabled is unset, maybeAutoBackup asks isBackupAutoEnabled -> primeHasGitRemote after every command, reads included. That built a RepoContext (`git -C .beads rev-parse --git-common-dir`) and ran `git remote`: two git processes per read even after the startup calls were gone. The throttle cannot short-circuit it: without a remote no backup ever runs, so there is no backup state to throttle on. Both probes are now answered in-process when provable, and the subprocess remains the fallback, so the decision and every warning/notice downstream of it are unchanged: - internal/beads getGitCommonDirForPath uses git.CommonDirInProcess (the discovery walk from the given directory, git's spelling). - primeHasGitRemote uses git.HasRemoteInProcess in the same directory primeGitCmd would run git in (primeGitDir, factored out). It answers only when every config file `git remote` reads -- system (GIT_CONFIG_SYSTEM, or /etc/gitconfig and /etc/gitconfig of the git on PATH), global (GIT_CONFIG_GLOBAL, or XDG and ~/.gitconfig) and the repository's own -- is absent or within scanGitConfig's grammar and include-free, no GIT_CONFIG_PARAMETERS/COUNT/GIT_CONFIG or discovery override is set, and no legacy .git/remotes or .git/branches entries exist. A remote seen only in a system-config candidate declines. Remote names outside [A-Za-z0-9._-] decline. Tests compare HasRemoteInProcess with `git remote` (no remote, local, worktree, submodule, remote.pushDefault only, global remote, no repo) and pin the declines (global include, malformed global, command-line config); CommonDirInProcess is compared with `git -C dir rev-parse --git-common-dir`. TestEmbeddedReadOnlyCommandsSpawnNoGit now runs with backup.enabled at its default, with and without a git remote, and still expects zero git processes for list/show. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/embedded-cmd-test-shards.txt | 1 + CHANGELOG.md | 16 + cmd/bd/BUILD.bazel | 1 + cmd/bd/actor_test.go | 54 ++ cmd/bd/ado.go | 6 +- cmd/bd/assign.go | 6 +- cmd/bd/audit.go | 8 +- cmd/bd/close.go | 10 +- cmd/bd/close_direct.go | 4 +- cmd/bd/close_if_revision.go | 12 +- cmd/bd/close_proxied_server.go | 10 +- cmd/bd/comment.go | 2 +- cmd/bd/comments.go | 2 +- cmd/bd/context.go | 11 +- cmd/bd/cook.go | 6 +- cmd/bd/create.go | 4 +- cmd/bd/create_form.go | 2 +- cmd/bd/defer.go | 2 +- cmd/bd/defer_proxied_server.go | 4 +- cmd/bd/delete.go | 4 +- cmd/bd/delete_proxied_server.go | 2 +- cmd/bd/dep.go | 10 +- cmd/bd/dep_proxied_server.go | 4 +- cmd/bd/duplicates.go | 2 +- cmd/bd/edit.go | 6 +- cmd/bd/gate.go | 12 +- cmd/bd/gate_discover.go | 2 +- cmd/bd/gate_proxied_server.go | 16 +- cmd/bd/gc_proxied_server.go | 2 +- cmd/bd/github.go | 2 +- cmd/bd/gitlab.go | 2 +- cmd/bd/graph_apply.go | 2 +- cmd/bd/heartbeat.go | 6 +- cmd/bd/heartbeat_proxied_server.go | 2 +- cmd/bd/human.go | 6 +- cmd/bd/human_proxied_server.go | 8 +- cmd/bd/jira.go | 2 +- cmd/bd/label.go | 8 +- cmd/bd/label_proxied_server.go | 6 +- cmd/bd/linear.go | 6 +- cmd/bd/link.go | 4 +- cmd/bd/link_proxied_server.go | 2 +- cmd/bd/main.go | 86 ++- cmd/bd/markdown.go | 4 +- cmd/bd/merge_slot.go | 8 +- cmd/bd/migrate_issues.go | 2 +- cmd/bd/mol_bond.go | 12 +- cmd/bd/mol_burn.go | 4 +- cmd/bd/mol_current.go | 2 +- cmd/bd/mol_progress.go | 2 +- cmd/bd/mol_proxied_server.go | 20 +- cmd/bd/mol_squash.go | 2 +- cmd/bd/mutate_proxied_server.go | 16 +- cmd/bd/note.go | 4 +- cmd/bd/notion.go | 2 +- cmd/bd/pour.go | 4 +- cmd/bd/prime.go | 29 +- cmd/bd/priority.go | 4 +- cmd/bd/promote.go | 4 +- cmd/bd/promote_proxied_server.go | 4 +- cmd/bd/purge.go | 2 +- cmd/bd/quick.go | 2 +- cmd/bd/quick_proxied_server.go | 2 +- cmd/bd/ready.go | 2 +- cmd/bd/ready_input.go | 2 +- cmd/bd/reclaim.go | 4 +- cmd/bd/relate.go | 8 +- cmd/bd/relate_proxied_server.go | 6 +- cmd/bd/rename_prefix.go | 4 +- cmd/bd/reopen.go | 4 +- cmd/bd/reopen_proxied_server.go | 4 +- cmd/bd/ship.go | 2 +- cmd/bd/startup_git_exec_embedded_test.go | 176 ++++++ cmd/bd/state.go | 4 +- cmd/bd/status.go | 2 +- cmd/bd/swarm.go | 16 +- cmd/bd/sync_push_pull.go | 26 +- cmd/bd/tag.go | 4 +- cmd/bd/unclaim.go | 6 +- cmd/bd/unclaim_proxied_server.go | 8 +- cmd/bd/undefer.go | 2 +- cmd/bd/update.go | 12 +- cmd/bd/update_proxied_server.go | 4 +- cmd/bd/wisp.go | 2 +- cmd/bd/wisp_proxied_server.go | 6 +- internal/beads/context.go | 20 +- internal/git/BUILD.bazel | 8 +- internal/git/discover_linux.go | 559 +++++++++++++++++++ internal/git/discover_linux_test.go | 471 ++++++++++++++++ internal/git/discover_other.go | 20 + internal/git/gitconfig_linux.go | 109 ++++ internal/git/gitdir.go | 51 +- 92 files changed, 1801 insertions(+), 233 deletions(-) create mode 100644 cmd/bd/startup_git_exec_embedded_test.go create mode 100644 internal/git/discover_linux.go create mode 100644 internal/git/discover_linux_test.go create mode 100644 internal/git/discover_other.go create mode 100644 internal/git/gitconfig_linux.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index b1c7b0e8c..7a1c26452 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -306,6 +306,7 @@ 50 17 TestEmbeddedCompact 50 17 TestEmbeddedGateConcurrent 50 17 TestEmbeddedListComments +50 17 TestEmbeddedReadOnlyCommandsSpawnNoGit 50 18 TestEmbeddedContext 50 18 TestEmbeddedContributorCreate diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d65701d..9831b832c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Every `bd` invocation used to start two git subprocesses before doing any + work: `git rev-parse` to locate the repository and `git config user.name` + to resolve the actor. On Linux the repository is now located in-process + (ordinary repositories, linked worktrees, submodules, no repository), + producing exactly what `git rev-parse` printed and still running git for + anything it does not model (`GIT_DIR` and related overrides, + `core.worktree`/`core.bare`, repository extensions, includes, foreign + ownership subject to `safe.directory`). The `git config user.name` actor + fallback is resolved only when a command actually needs an actor, so + read-only commands such as `bd list` and `bd show` start no git process + for it. The actor's value and priority order are unchanged. In an embedded + workspace with `backup.enabled` unset, the per-command auto-backup check + ("is there a git remote?") is likewise answered from the git config files + when they are plain (no includes, no command-line config), instead of + running `git -C .beads rev-parse --git-common-dir` and `git remote`. + - `bd create --graph` now stores a plan whose path from a node's parent to the node runs through a `waits-for` edge; the graph-only preflight that walked every ready-work edge used to refuse it. The plan now goes through diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 8f5ea468f..5964f512a 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1147,6 +1147,7 @@ go_test( "sql_test.go", "stale_embedded_test.go", "stale_test.go", + "startup_git_exec_embedded_test.go", "state_embedded_test.go", "state_proxied_integration_test.go", "state_test.go", diff --git a/cmd/bd/actor_test.go b/cmd/bd/actor_test.go index d7efa7fc3..5ef83b505 100644 --- a/cmd/bd/actor_test.go +++ b/cmd/bd/actor_test.go @@ -246,3 +246,57 @@ func TestGetActorWithGit_PriorityOrder(t *testing.T) { t.Errorf("Expected BD_ACTOR to be used as fallback, got %q", result) } } + +// TestDeferredActorGitFallback pins the lazy actor: PersistentPreRunE defers +// the git config user.name fallback, and the first reader resolves it to +// exactly what the former eager getActorWithGit assignment produced. +func TestDeferredActorGitFallback(t *testing.T) { + origActor, origPending := actor, actorGitFallbackPending + var origCtxActor string + if cmdCtx != nil { + origCtxActor = cmdCtx.Actor + } + t.Cleanup(func() { + actor, actorGitFallbackPending = origActor, origPending + if cmdCtx != nil { + cmdCtx.Actor = origCtxActor + } + }) + t.Setenv("BEADS_ACTOR", "") + t.Setenv("BD_ACTOR", "") + + actor = "explicit" + deferActorGitFallback() + if actorGitFallbackPending { + t.Fatal("a configured actor must not defer a git lookup") + } + if got := currentActor(); got != "explicit" { + t.Fatalf("currentActor() = %q, want %q", got, "explicit") + } + + actor = "" + want := resolveActorWithGit() + deferActorGitFallback() + if !actorGitFallbackPending || actor != "" { + t.Fatalf("deferral resolved eagerly: pending=%v actor=%q", actorGitFallbackPending, actor) + } + if got := currentActor(); got != want { + t.Fatalf("currentActor() = %q, want eager resolution %q", got, want) + } + if actorGitFallbackPending || actor != want { + t.Fatalf("after first use: pending=%v actor=%q, want resolved %q", actorGitFallbackPending, actor, want) + } + + actor = "" + deferActorGitFallback() + if got := getActorWithGit(); got != want || actorGitFallbackPending || actor != want { + t.Fatalf("getActorWithGit() = %q (pending=%v actor=%q), want %q resolved into the global", got, actorGitFallbackPending, actor, want) + } + + actor = "" + deferActorGitFallback() + setActor("assigned") + if got := currentActor(); got != "assigned" { + t.Fatalf("setActor must cancel a deferred fallback: currentActor() = %q", got) + } +} diff --git a/cmd/bd/ado.go b/cmd/bd/ado.go index 08206992b..5e54257bc 100644 --- a/cmd/bd/ado.go +++ b/cmd/bd/ado.go @@ -540,7 +540,7 @@ func runADOSync(cmd *cobra.Command, _ []string) error { } // Create the sync engine - engine := tracker.NewEngine(at, store, actor) + engine := tracker.NewEngine(at, store, currentActor()) var warnings []string if !jsonOutput { engine.OnMessage = func(msg string) { _, _ = fmt.Fprintln(out, " "+msg) } @@ -641,7 +641,7 @@ func runADOSync(cmd *cobra.Command, _ []string) error { continue } reason := fmt.Sprintf("ADO work item %s deleted", idStr) - if cerr := store.CloseIssue(ctx, localID, reason, actor, ""); cerr != nil { + if cerr := store.CloseIssue(ctx, localID, reason, currentActor(), ""); cerr != nil { msg := fmt.Sprintf("Failed to close %s for deleted ADO #%s: %v", localID, idStr, cerr) warnings = append(warnings, msg) if !jsonOutput { @@ -915,7 +915,7 @@ func buildADOPullHooks(ctx context.Context, at *ado.Tracker, bootstrapMatch, noC "external_ref": ref, "source_system": "ado:" + extIssue.ID, } - if err := store.UpdateIssue(ctx, result.BeadsID, updates, actor); err == nil { + if err := store.UpdateIssue(ctx, result.BeadsID, updates, currentActor()); err == nil { *matchCount++ if warn != nil { warn(fmt.Sprintf("Bootstrap matched ADO #%s → %s (%s)", extIssue.ID, result.BeadsID, result.MatchType)) diff --git a/cmd/bd/assign.go b/cmd/bd/assign.go index ba3a72e97..83b145f33 100644 --- a/cmd/bd/assign.go +++ b/cmd/bd/assign.go @@ -85,7 +85,7 @@ Examples: // lost race reports precondition_failed from the guarded write below // instead of this policy refusal — see ifRevisionAlreadyStale's doc. if !ifRevisionAlreadyStale(result.Issue, ifRevision) { - if err := validateIssueReassignable(id, result.Issue, actor, assignee, + if err := validateIssueReassignable(id, result.Issue, currentActor(), assignee, storeClaimPoolAliases(ctx, issueStore), force); err != nil { return HandleErrorRespectJSON("%s", err) } @@ -104,7 +104,7 @@ Examples: return HandleErrorRespectJSON("%v", err) } mutationResult, err := runCommandUpdateMutation(opsCtx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: result.ResolvedID, patch: issueops.IssuePatch{ Assignee: issueops.Field[string]{Set: true, Value: assignee}, @@ -121,7 +121,7 @@ Examples: return HandleErrorRespectJSON("updating %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "assign", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/audit.go b/cmd/bd/audit.go index 127e3f236..fd33d0536 100644 --- a/cmd/bd/audit.go +++ b/cmd/bd/audit.go @@ -77,8 +77,8 @@ var auditRecordCmd = &cobra.Command{ if err := json.Unmarshal(b, &e); err != nil { return HandleError("invalid JSON on stdin: %v", err) } - if actor != "" { - e.Actor = actor + if currentActor() != "" { + e.Actor = currentActor() } } else { if auditRecordKind == "" { @@ -86,7 +86,7 @@ var auditRecordCmd = &cobra.Command{ } e = audit.Entry{ Kind: auditRecordKind, - Actor: actor, + Actor: currentActor(), IssueID: auditRecordIssueID, Model: auditRecordModel, Prompt: auditRecordPrompt, @@ -137,7 +137,7 @@ var auditLabelCmd = &cobra.Command{ } e := audit.Entry{ Kind: "label", - Actor: actor, + Actor: currentActor(), ParentID: parentID, Label: auditLabelValue, Reason: auditLabelReason, diff --git a/cmd/bd/close.go b/cmd/bd/close.go index b99abe219..5a28910a2 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -227,7 +227,7 @@ the flags appear in the command line.`, // suppressed real-close side effects (no audit, no closed→closed on the // step). Register the store when it actually closed the root so the // pending-commit sweep persists it — closedCount==0 would not commit. - if molID := autoCloseCompletedMolecule(ctx, activeStore, id, actor, session); molID != "" { + if molID := autoCloseCompletedMolecule(ctx, activeStore, id, currentActor(), session); molID != "" { mutatedStores[activeStore] = append(mutatedStores[activeStore], molID) } } else { @@ -238,13 +238,13 @@ the flags appear in the command line.`, if issue != nil { oldStatus = string(issue.Status) } - audit.LogFieldChange(id, "status", oldStatus, "closed", actor, reason) + audit.LogFieldChange(id, "status", oldStatus, "closed", currentActor(), reason) closedCount++ // Auto-close parent molecule if all steps are now complete. // Runs against the same store the step was closed in. - autoCloseCompletedMolecule(ctx, activeStore, id, actor, session) + autoCloseCompletedMolecule(ctx, activeStore, id, currentActor(), session) } // First id this command settled as closed — a real close or an @@ -313,7 +313,7 @@ the flags appear in the command line.`, if continueFlag && len(resolvedIDs) == 1 && closedForCommand { autoClaim := !noAuto - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(postCloseStore), resolvedIDs[0], autoClaim, actor) + result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(postCloseStore), resolvedIDs[0], autoClaim, currentActor()) if err != nil { fmt.Fprintf(os.Stderr, "Warning: could not advance to next step: %v\n", err) } else if result != nil { @@ -400,7 +400,7 @@ the flags appear in the command line.`, if s == nil { continue } - if err := commitPendingIfEmbedded(ctx, s, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, s, currentActor(), doltAutoCommitParams{ Command: "close", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/close_direct.go b/cmd/bd/close_direct.go index d7ac30cb1..47890945e 100644 --- a/cmd/bd/close_direct.go +++ b/cmd/bd/close_direct.go @@ -81,7 +81,7 @@ func closeDirectCheckOne(id string, issue *types.Issue, st storage.DoltStorage, // guard remains the authority on whether the close is a no-op, so a // concurrent close still converges. Mirrored in closeProxiedCheckOne. if issue == nil || issue.Status != types.StatusClosed { - if err := validateIssueClosable(id, issue, actor, force); err != nil { + if err := validateIssueClosable(id, issue, currentActor(), force); err != nil { return err.Error() } } @@ -135,7 +135,7 @@ func closeDirectBatches(items []closeDirectItem) []closeDirectBatch { // hands out ONE claim however many stores the ids spanned. func closeDirectRequest(batch closeDirectBatch, session string, force bool, claimStore storage.DoltStorage, claimNext *issueops.ReadyRequest) issueops.CloseBatchRequest { request := issueops.CloseBatchRequest{ - Actor: actor, + Actor: currentActor(), Items: make([]issueops.BatchCloseItem, 0, len(batch.items)), Session: session, Force: force, diff --git a/cmd/bd/close_if_revision.go b/cmd/bd/close_if_revision.go index ac5bfda1a..b544714e5 100644 --- a/cmd/bd/close_if_revision.go +++ b/cmd/bd/close_if_revision.go @@ -50,7 +50,7 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool preCloseStatus := string(result.Issue.Status) closeResult, closeErr := ops.Close(opsCtx, issueops.CloseRequest{ - Actor: actor, + Actor: currentActor(), IssueID: result.ResolvedID, Reason: reason, Session: session, @@ -79,11 +79,11 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool // entirely (mc-zndi7.75) and would otherwise leave a molecule's root // stranded open forever. mutatedIDs := []string{result.ResolvedID} - if molID := autoCloseCompletedMolecule(ctx, result.Store, result.ResolvedID, actor, session); molID != "" { + if molID := autoCloseCompletedMolecule(ctx, result.Store, result.ResolvedID, currentActor(), session); molID != "" { mutatedIDs = append(mutatedIDs, molID) } - if err := commitPendingIfEmbedded(ctx, result.Store, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, result.Store, currentActor(), doltAutoCommitParams{ Command: "close", IssueIDs: mutatedIDs, }); err != nil { @@ -116,7 +116,7 @@ func runCloseProxiedIfRevision(ctx context.Context, id, reason string, force boo } closeResult, closeErr := ops.Close(ctx, issueops.CloseRequest{ - Actor: actor, + Actor: currentActor(), IssueID: id, Reason: reason, Session: session, @@ -165,7 +165,7 @@ func runCloseIfRevisionProxiedPostClose(ctx context.Context, id, session string) } autoClosedMol, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*types.Issue, string, error) { var warnings []string - mol := autoCloseProxiedCompletedMolecule(ctx, uw, id, actor, session, &warnings) + mol := autoCloseProxiedCompletedMolecule(ctx, uw, id, currentActor(), session, &warnings) for _, w := range warnings { fmt.Fprintf(os.Stderr, "Warning: %s\n", w) } @@ -209,7 +209,7 @@ func reportClosedIfRevisionResult(id, reason, preCloseStatus string, closeResult closedIssue.Dependencies = nil } if closeResult.Changed { - audit.LogFieldChange(id, "status", preCloseStatus, "closed", actor, reason) + audit.LogFieldChange(id, "status", preCloseStatus, "closed", currentActor(), reason) } if jsonOutput { if closedIssue != nil { diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index 94a614e3a..e7a791d4f 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -140,7 +140,7 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin return HandleErrorRespectJSON("%v", cerr) } result, err = closer.CloseBatch(ctx, issueops.CloseBatchRequest{ - Actor: actor, + Actor: currentActor(), Items: pre.items, Session: in.session, Force: in.force, @@ -166,7 +166,7 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin for i, o := range outcomes { if o.closed { - audit.LogFieldChange(o.id, "status", o.auditOld, "closed", actor, o.auditReason) + audit.LogFieldChange(o.id, "status", o.auditOld, "closed", currentActor(), o.auditReason) } if !in.jsonOut { fmt.Printf("%s Closed %s: %s\n", ui.RenderPass("✓"), formatFeedbackID(o.after.ID, o.after.Title), closeReasons[i]) @@ -300,7 +300,7 @@ func closeProxiedCheckOne(ctx context.Context, uw uow.UnitOfWork, id string, in // always been. Both close paths must agree here — diverging is the defect // class #5217 closed. if current.Status != types.StatusClosed { - if err := validateIssueClosable(id, current, actor, in.force); err != nil { + if err := validateIssueClosable(id, current, currentActor(), in.force); err != nil { return err.Error(), nil } } @@ -483,7 +483,7 @@ func closeProxiedRunPostClose(ctx context.Context, args []string, in closeProxie var wrote []string for _, o := range outcomes { - mol := autoCloseProxiedCompletedMolecule(ctx, uw, o.id, actor, in.session, &out.warnings) + mol := autoCloseProxiedCompletedMolecule(ctx, uw, o.id, currentActor(), in.session, &out.warnings) if mol != nil { out.autoClosedMol = mol wrote = append(wrote, "auto-close "+mol.ID) @@ -529,7 +529,7 @@ func closeProxiedSuggestNext(ctx context.Context, uw uow.UnitOfWork, closedID st } func closeProxiedContinue(ctx context.Context, uw uow.UnitOfWork, closedID string, autoClaim bool) (*ContinueResult, string) { - result, err := AdvanceToNextStep(ctx, newUOWMolWriter(uw), closedID, autoClaim, actor) + result, err := AdvanceToNextStep(ctx, newUOWMolWriter(uw), closedID, autoClaim, currentActor()) if err != nil { return nil, fmt.Sprintf("could not advance to next step: %v", err) } diff --git a/cmd/bd/comment.go b/cmd/bd/comment.go index 15f577e7c..76cf3342a 100644 --- a/cmd/bd/comment.go +++ b/cmd/bd/comment.go @@ -183,7 +183,7 @@ To list comments on an issue, use the plural form: bd comments `, if err != nil { return HandleErrorRespectJSON("adding comment: %v", err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "comment", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/comments.go b/cmd/bd/comments.go index c789ef498..256e6151f 100644 --- a/cmd/bd/comments.go +++ b/cmd/bd/comments.go @@ -258,7 +258,7 @@ Examples: if err != nil { return HandleErrorRespectJSON("adding comment: %v", err) } - if err := commitPendingIfEmbedded(ctx, result.Store, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, result.Store, currentActor(), doltAutoCommitParams{ Command: "comments add", IssueIDs: []string{issueID}, }); err != nil { diff --git a/cmd/bd/context.go b/cmd/bd/context.go index 6b4807e47..bd772896b 100644 --- a/cmd/bd/context.go +++ b/cmd/bd/context.go @@ -112,17 +112,26 @@ func setStore(s storage.DoltStorage) { // getActor returns the current actor name for audit trail. func getActor() string { if shouldUseGlobals() { - return actor + return currentActor() + } + // cmdCtx.Actor is a copy of the global taken by syncCommandContext or + // setActor. While the two agree, the global is authoritative and may + // still carry a deferred git fallback (see deferActorGitFallback). + if cmdCtx.Actor == actor { + return currentActor() } return cmdCtx.Actor } // setActor updates the actor name in the CommandContext. func setActor(a string) { + actorMu.Lock() + defer actorMu.Unlock() if cmdCtx != nil { cmdCtx.Actor = a } actor = a + actorGitFallbackPending = false } // isJSONOutput returns true if JSON output mode is enabled. diff --git a/cmd/bd/cook.go b/cmd/bd/cook.go index d76422c97..6e94309b0 100644 --- a/cmd/bd/cook.go +++ b/cmd/bd/cook.go @@ -1001,20 +1001,20 @@ func cookFormula(ctx context.Context, s storage.DoltStorage, f *formula.Formula, } // Create all issues - if err := tx.CreateIssues(ctx, issues, actor); err != nil { + if err := tx.CreateIssues(ctx, issues, currentActor()); err != nil { return fmt.Errorf("failed to create issues: %w", err) } // Add labels for _, l := range labels { - if err := tx.AddLabel(ctx, l.issueID, l.label, actor); err != nil { + if err := tx.AddLabel(ctx, l.issueID, l.label, currentActor()); err != nil { return fmt.Errorf("failed to add label %s to %s: %w", l.label, l.issueID, err) } } // Add dependencies for _, dep := range deps { - if err := tx.AddDependency(ctx, dep, actor); err != nil { + if err := tx.AddDependency(ctx, dep, currentActor()); err != nil { return fmt.Errorf("failed to create dependency: %w", err) } } diff --git a/cmd/bd/create.go b/cmd/bd/create.go index e65c9ab82..38692d287 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -568,7 +568,7 @@ var createCmd = &cobra.Command{ // dry-run preview needs it too; asking the facade to inherit as well // would append the parent's labels a second time. result, err := ops.Create(opsCtx, issueops.CreateRequest{ - Actor: actor, + Actor: currentActor(), Issue: issue, ParentID: parentID, Dependencies: createDependencyRequests(depSpecs), @@ -610,7 +610,7 @@ var createCmd = &cobra.Command{ } if repoPath != "." && targetStore != nil { - if err := commitPendingIfEmbedded(ctx, targetStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, targetStore, currentActor(), doltAutoCommitParams{ Command: "create", IssueIDs: []string{created.ID}, }); err != nil { diff --git a/cmd/bd/create_form.go b/cmd/bd/create_form.go index d52f88dce..42d455796 100644 --- a/cmd/bd/create_form.go +++ b/cmd/bd/create_form.go @@ -371,7 +371,7 @@ func runCreateForm(cmd *cobra.Command) error { fv := parseCreateFormInput(raw) fv.ParentID = parentID - issue, err := CreateIssueFromFormValues(rootCtx, store, fv, actor) + issue, err := CreateIssueFromFormValues(rootCtx, store, fv, currentActor()) if err != nil { return HandleError("%v", err) } diff --git a/cmd/bd/defer.go b/cmd/bd/defer.go index 4158253bb..0a22b18be 100644 --- a/cmd/bd/defer.go +++ b/cmd/bd/defer.go @@ -123,7 +123,7 @@ Examples: updates["notes"] = notes + reason } - if err := store.UpdateIssue(ctx, fullID, updates, actor); err != nil { + if err := store.UpdateIssue(ctx, fullID, updates, currentActor()); err != nil { fmt.Fprintf(os.Stderr, "Error deferring %s: %v\n", fullID, err) continue } diff --git a/cmd/bd/defer_proxied_server.go b/cmd/bd/defer_proxied_server.go index 280df6ace..0e1a47847 100644 --- a/cmd/bd/defer_proxied_server.go +++ b/cmd/bd/defer_proxied_server.go @@ -27,9 +27,9 @@ type deferProxiedResult struct { func proxiedUpdateByID(ctx context.Context, uw uow.UnitOfWork, id string, isWisp bool, updates map[string]any) error { if isWisp { - return uw.IssueUseCase().UpdateWisp(ctx, id, updates, actor) + return uw.IssueUseCase().UpdateWisp(ctx, id, updates, currentActor()) } - return uw.IssueUseCase().UpdateIssue(ctx, id, updates, actor) + return uw.IssueUseCase().UpdateIssue(ctx, id, updates, currentActor()) } func proxiedGetByID(ctx context.Context, uw uow.UnitOfWork, id string, isWisp bool) *types.Issue { diff --git a/cmd/bd/delete.go b/cmd/bd/delete.go index d48de774a..c767f3958 100644 --- a/cmd/bd/delete.go +++ b/cmd/bd/delete.go @@ -163,7 +163,7 @@ Force: Delete and orphan dependents // rewrite runs INSIDE the transaction that deletes, because it is the // role's. request := issueops.DeleteRequest{ - Actor: actor, + Actor: currentActor(), IDs: []string{issueID}, Force: force, DryRun: dryRun || !force, @@ -378,7 +378,7 @@ func deleteBatch(_ *cobra.Command, issueIDs []string, force bool, dryRun bool, c // --force is the confirmation as well as the orphan mode, so an unconfirmed // run asks the role what it WOULD do; see the single-id path. request := issueops.DeleteRequest{ - Actor: actor, + Actor: currentActor(), IDs: resolvedIDs, Cascade: cascade, Force: force, diff --git a/cmd/bd/delete_proxied_server.go b/cmd/bd/delete_proxied_server.go index 9cd6b0491..3fff3137d 100644 --- a/cmd/bd/delete_proxied_server.go +++ b/cmd/bd/delete_proxied_server.go @@ -102,7 +102,7 @@ func runDeleteProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri // --force is the confirmation as well as the orphan mode, exactly as on the // direct route, so an unconfirmed run asks the role what it WOULD do. request := issueops.DeleteRequest{ - Actor: actor, + Actor: currentActor(), IDs: in.ids, Cascade: in.cascade, Force: in.force, diff --git a/cmd/bd/dep.go b/cmd/bd/dep.go index ef502d810..e77e1e1c8 100644 --- a/cmd/bd/dep.go +++ b/cmd/bd/dep.go @@ -37,7 +37,7 @@ func addDependencyEdgesDirect(ctx context.Context, st storage.DoltStorage, edges return err } _, err = editor.AddDependencies(ctx, issueops.AddDependenciesRequest{ - Actor: actor, + Actor: currentActor(), Edges: edges, SkipPerEdgeCycleCheck: skipPerEdgeCycleCheck, }) @@ -330,7 +330,7 @@ Examples: warnIfCyclesExist(fromStore) } - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "dep add", IssueIDs: []string{fromID, toID}, }); err != nil { @@ -520,7 +520,7 @@ Examples: warnIfCyclesExist(fromStore) } - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "dep add", IssueIDs: []string{fromID, toID}, }); err != nil { @@ -1270,7 +1270,7 @@ var depRemoveCmd = &cobra.Command{ return HandleErrorRespectJSON("%v", err) } result, err := editor.RemoveDependency(opsCtx, issueops.RemoveDependencyRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fullFromID, DependsOnID: fullToID, }) @@ -1278,7 +1278,7 @@ var depRemoveCmd = &cobra.Command{ return HandleErrorRespectJSON("%v", err) } - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "dep remove", IssueIDs: []string{fullFromID, fullToID}, }); err != nil { diff --git a/cmd/bd/dep_proxied_server.go b/cmd/bd/dep_proxied_server.go index 28e50f79f..aee6b0306 100644 --- a/cmd/bd/dep_proxied_server.go +++ b/cmd/bd/dep_proxied_server.go @@ -76,7 +76,7 @@ func addDependencyEdgesProxied(ctx context.Context, edges []issueops.DependencyE return err } _, err = editor.AddDependencies(ctx, issueops.AddDependenciesRequest{ - Actor: actor, + Actor: currentActor(), Edges: edges, SkipPerEdgeCycleCheck: skipPerEdgeCycleCheck, }) @@ -370,7 +370,7 @@ func runDepRemoveProxiedServer(_ *cobra.Command, ctx context.Context, args []str return HandleErrorRespectJSON("%v", err) } result, err := editor.RemoveDependency(ctx, issueops.RemoveDependencyRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fromID, DependsOnID: toID, }) diff --git a/cmd/bd/duplicates.go b/cmd/bd/duplicates.go index 32c6417c0..f11e4ea35 100644 --- a/cmd/bd/duplicates.go +++ b/cmd/bd/duplicates.go @@ -424,7 +424,7 @@ func performMerge(targetID string, sourceIDs []string) map[string]interface{} { // Close the duplicate issue reason := fmt.Sprintf("Duplicate of %s", targetID) - if err := store.CloseIssue(ctx, sourceID, reason, actor, ""); err != nil { + if err := store.CloseIssue(ctx, sourceID, reason, currentActor(), ""); err != nil { errors = append(errors, fmt.Sprintf("failed to close %s: %v", sourceID, err)) continue } diff --git a/cmd/bd/edit.go b/cmd/bd/edit.go index 5fe2bed53..5e04128d0 100644 --- a/cmd/bd/edit.go +++ b/cmd/bd/edit.go @@ -150,7 +150,7 @@ Examples: fieldToEdit: newValue, } - err = issueStore.UpdateIssue(ctx, id, updates, actor) + err = issueStore.UpdateIssue(ctx, id, updates, currentActor()) if err != nil { if accessor, ok := storage.UnwrapStore(issueStore).(storage.RawDBAccessor); ok { if pingErr := accessor.DB().PingContext(ctx); pingErr != nil { @@ -158,14 +158,14 @@ Examples: _ = accessor.DB().PingContext(ctx) } } - err = issueStore.UpdateIssue(ctx, id, updates, actor) + err = issueStore.UpdateIssue(ctx, id, updates, currentActor()) } if err != nil { fmt.Fprintf(os.Stderr, "Your edits are preserved in: %s\n", tmpPath) return HandleErrorRespectJSON("updating issue: %v", err) } editSaved = true - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "edit", IssueIDs: []string{id}, }); err != nil { diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 3aa97289e..98985e062 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -275,7 +275,7 @@ This is used by 'bd done --phase-complete' to register for gate wake notificatio updates := map[string]interface{}{ "waiters": newWaiters, } - if err := store.UpdateIssue(ctx, gateID, updates, actor); err != nil { + if err := store.UpdateIssue(ctx, gateID, updates, currentActor()); err != nil { return HandleError("updating gate: %v", err) } @@ -355,7 +355,7 @@ Examples: } gate.Metadata = metadata - if err := store.CreateIssue(ctx, gate, actor); err != nil { + if err := store.CreateIssue(ctx, gate, currentActor()); err != nil { return HandleErrorRespectJSON("creating gate: %v", err) } @@ -364,7 +364,7 @@ Examples: DependsOnID: gate.ID, Type: types.DepBlocks, } - if err := store.AddDependency(ctx, dep, actor); err != nil { + if err := store.AddDependency(ctx, dep, currentActor()); err != nil { return HandleErrorRespectJSON("adding blocking dependency: %v", err) } @@ -573,7 +573,7 @@ Use --reason to provide context for why the gate was resolved.`, return HandleError("%s is not a gate issue (type=%s)", gateID, issue.IssueType) } - if err := store.CloseIssue(ctx, gateID, reason, actor, ""); err != nil { + if err := store.CloseIssue(ctx, gateID, reason, currentActor(), ""); err != nil { return HandleError("closing gate: %v", err) } @@ -679,7 +679,7 @@ Examples: return updateGateAwaitIDFunc(nil, gateID, runID) } recordSeen = func(gate *types.Issue) error { - if err := store.UpdateIssue(ctx, gate.ID, beadGateSeenUpdate(gate.AwaitID), actor); err != nil { + if err := store.UpdateIssue(ctx, gate.ID, beadGateSeenUpdate(gate.AwaitID), currentActor()); err != nil { return err } commandDidWrite.Store(true) @@ -1575,7 +1575,7 @@ func beadGateRetargetUpdate(awaitID string, dropSeen bool) map[string]interface{ // closeGate closes a gate issue with the given reason func closeGate(_ interface{}, gateID, reason string) error { - if err := store.CloseIssue(rootCtx, gateID, reason, actor, ""); err != nil { + if err := store.CloseIssue(rootCtx, gateID, reason, currentActor(), ""); err != nil { return err } commandDidWrite.Store(true) diff --git a/cmd/bd/gate_discover.go b/cmd/bd/gate_discover.go index f66c3d4fa..b1842e1de 100644 --- a/cmd/bd/gate_discover.go +++ b/cmd/bd/gate_discover.go @@ -581,7 +581,7 @@ func updateGateAwaitID(_ interface{}, gateID, runID string) error { updates := map[string]interface{}{ "await_id": runID, } - if err := store.UpdateIssue(rootCtx, gateID, updates, actor); err != nil { + if err := store.UpdateIssue(rootCtx, gateID, updates, currentActor()); err != nil { return err } return nil diff --git a/cmd/bd/gate_proxied_server.go b/cmd/bd/gate_proxied_server.go index 8597a749b..f83e66186 100644 --- a/cmd/bd/gate_proxied_server.go +++ b/cmd/bd/gate_proxied_server.go @@ -140,7 +140,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { } for gateID, runID := range discovered { - if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"await_id": runID}, actor); err != nil { + if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"await_id": runID}, currentActor()); err != nil { out.awaitErrs[gateID] = fmt.Errorf("failed to update gate with discovered run ID: %w", err) continue } @@ -168,7 +168,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { if before != nil && before.Status == types.StatusClosed { continue } - res, closeErr := uw.IssueUseCase().CloseIssue(ctx, r.gate.ID, domain.CloseIssueParams{Reason: r.reason}, actor) + res, closeErr := uw.IssueUseCase().CloseIssue(ctx, r.gate.ID, domain.CloseIssueParams{Reason: r.reason}, currentActor()) if closeErr != nil { out.closeErrs[r.gate.ID] = closeErr continue @@ -192,7 +192,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { } for _, c := range applied.closed { - audit.LogFieldChange(c.after.ID, "status", c.oldStatus, "closed", actor, c.reason) + audit.LogFieldChange(c.after.ID, "status", c.oldStatus, "closed", currentActor(), c.reason) } if len(applied.closed) > 0 || len(applied.updated) > 0 { commandDidWrite.Store(true) @@ -326,7 +326,7 @@ func runGateAddWaiterProxiedServer(_ *cobra.Command, ctx context.Context, args [ } newWaiters := append(issue.Waiters, waiter) - if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"waiters": newWaiters}, actor); err != nil { + if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"waiters": newWaiters}, currentActor()); err != nil { return out, "", fmt.Errorf("updating gate: %w", err) } if after, getErr := uw.IssueUseCase().GetIssue(ctx, gateID); getErr == nil { @@ -394,7 +394,7 @@ func runGateCreateProxiedServer(cmd *cobra.Command, ctx context.Context) error { } gate.Metadata = metadata - res, err := uw.IssueUseCase().CreateIssue(ctx, domain.CreateIssueParams{Issue: gate}, actor) + res, err := uw.IssueUseCase().CreateIssue(ctx, domain.CreateIssueParams{Issue: gate}, currentActor()) if err != nil { return out, "", fmt.Errorf("creating gate: %w", err) } @@ -404,7 +404,7 @@ func runGateCreateProxiedServer(cmd *cobra.Command, ctx context.Context) error { DependsOnID: res.Issue.ID, Type: types.DepBlocks, } - if err := uw.DependencyUseCase().AddDependency(ctx, dep, actor); err != nil { + if err := uw.DependencyUseCase().AddDependency(ctx, dep, currentActor()); err != nil { return out, "", fmt.Errorf("adding blocking dependency: %w", err) } @@ -464,7 +464,7 @@ func runGateResolveProxiedServer(cmd *cobra.Command, ctx context.Context, args [ return out, "", fmt.Errorf("%s is not a gate issue (type=%s)", gateID, issue.IssueType) } - res, err := uw.IssueUseCase().CloseIssue(ctx, gateID, domain.CloseIssueParams{Reason: reason}, actor) + res, err := uw.IssueUseCase().CloseIssue(ctx, gateID, domain.CloseIssueParams{Reason: reason}, currentActor()) if err != nil { return out, "", fmt.Errorf("closing gate: %w", err) } @@ -486,7 +486,7 @@ func runGateResolveProxiedServer(cmd *cobra.Command, ctx context.Context, args [ // double-resolve must not re-log it (same guard as the o.closed check in // close_proxied_server.go). if applied.closed && applied.after != nil { - audit.LogFieldChange(applied.after.ID, "status", applied.oldStatus, "closed", actor, reason) + audit.LogFieldChange(applied.after.ID, "status", applied.oldStatus, "closed", currentActor(), reason) } commandDidWrite.Store(true) diff --git a/cmd/bd/gc_proxied_server.go b/cmd/bd/gc_proxied_server.go index ed4818373..f5fbdb7df 100644 --- a/cmd/bd/gc_proxied_server.go +++ b/cmd/bd/gc_proxied_server.go @@ -93,7 +93,7 @@ func runGCProxiedServer(ctx context.Context) error { ids[i] = issue.ID } deleteResult, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (domain.DeleteIssuesResult, string, error) { - res, err := uw.IssueUseCase().DeleteIssues(ctx, domain.DeleteIssuesParams{IDs: ids}, actor) + res, err := uw.IssueUseCase().DeleteIssues(ctx, domain.DeleteIssuesParams{IDs: ids}, currentActor()) if err != nil { return domain.DeleteIssuesResult{}, "", err } diff --git a/cmd/bd/github.go b/cmd/bd/github.go index 77ee4e8e8..bedc3ef4b 100644 --- a/cmd/bd/github.go +++ b/cmd/bd/github.go @@ -419,7 +419,7 @@ func runGitHubSync(cmd *cobra.Command, args []string) error { } // Create the sync engine - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.OnMessage = func(msg string) { _, _ = fmt.Fprintln(out, " "+msg) } engine.OnWarning = func(msg string) { _, _ = fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } diff --git a/cmd/bd/gitlab.go b/cmd/bd/gitlab.go index 6657dcb3b..6bc94e0d0 100644 --- a/cmd/bd/gitlab.go +++ b/cmd/bd/gitlab.go @@ -511,7 +511,7 @@ func runGitLabSync(cmd *cobra.Command, args []string) error { } // Create the sync engine - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) if !jsonOutput { engine.OnMessage = func(msg string) { _, _ = fmt.Fprintln(out, " "+msg) } } diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 00faeb197..9a52f1415 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -1167,7 +1167,7 @@ func graphApplyEdgeBatchMetadata(edge GraphApplyEdge, depType types.DependencyTy } func executeGraphApply(ctx context.Context, plan *GraphApplyPlan, opts GraphApplyOptions) (*GraphApplyResult, error) { - req, err := buildGraphApplyBatchRequest(plan, opts, actor, getOwner()) + req, err := buildGraphApplyBatchRequest(plan, opts, currentActor(), getOwner()) if err != nil { return nil, err } diff --git a/cmd/bd/heartbeat.go b/cmd/bd/heartbeat.go index 281db80c2..6790047a7 100644 --- a/cmd/bd/heartbeat.go +++ b/cmd/bd/heartbeat.go @@ -68,11 +68,11 @@ Examples: defer result.Close() issueStore := result.Store - if err := issueStore.HeartbeatIssue(ctx, result.ResolvedID, actor); err != nil { + if err := issueStore.HeartbeatIssue(ctx, result.ResolvedID, currentActor()); err != nil { return HandleErrorRespectJSON("heartbeat %s: %v", result.ResolvedID, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "heartbeat", IssueIDs: []string{result.ResolvedID}, }); err != nil { @@ -94,7 +94,7 @@ func renderHeartbeatSuccess(id, title string) error { return outputJSON(map[string]string{ "id": id, "status": "heartbeat", - "owner": actor, + "owner": currentActor(), }) } fmt.Printf("%s Heartbeat %s (lease refreshed)\n", ui.RenderPass("✓"), formatFeedbackID(id, title)) diff --git a/cmd/bd/heartbeat_proxied_server.go b/cmd/bd/heartbeat_proxied_server.go index 80de4a033..c09772f52 100644 --- a/cmd/bd/heartbeat_proxied_server.go +++ b/cmd/bd/heartbeat_proxied_server.go @@ -41,7 +41,7 @@ func runHeartbeatProxiedServer(ctx context.Context, id string) error { } // Wisps resolve here too and are refused below: the repo verb // classifies them ErrNotClaimable ("is ephemeral"), same as classic. - if herr := uw.IssueUseCase().Heartbeat(ctx, issue.ID, actor); herr != nil { + if herr := uw.IssueUseCase().Heartbeat(ctx, issue.ID, currentActor()); herr != nil { return heartbeatProxiedOutcome{}, fmt.Errorf("heartbeat %s: %w", issue.ID, herr) } return heartbeatProxiedOutcome{id: issue.ID, title: issue.Title}, nil diff --git a/cmd/bd/human.go b/cmd/bd/human.go index c87faa2d6..19fb98907 100644 --- a/cmd/bd/human.go +++ b/cmd/bd/human.go @@ -307,12 +307,12 @@ Examples: warnIfNotHumanLabeled(issue) - _, err = targetStore.AddIssueComment(ctx, resolvedID, actor, commentText) + _, err = targetStore.AddIssueComment(ctx, resolvedID, currentActor(), commentText) if err != nil { return HandleErrorRespectJSON("adding comment: %v", err) } - if err := targetStore.CloseIssue(ctx, resolvedID, "Responded", actor, ""); err != nil { + if err := targetStore.CloseIssue(ctx, resolvedID, "Responded", currentActor(), ""); err != nil { return HandleErrorRespectJSON("closing bead: %v", err) } @@ -403,7 +403,7 @@ Examples: warnIfNotHumanLabeled(issue) - if err := targetStore.CloseIssue(ctx, resolvedID, closeReason, actor, ""); err != nil { + if err := targetStore.CloseIssue(ctx, resolvedID, closeReason, currentActor(), ""); err != nil { return HandleErrorRespectJSON("closing bead: %v", err) } diff --git a/cmd/bd/human_proxied_server.go b/cmd/bd/human_proxied_server.go index 10c948add..924ed2632 100644 --- a/cmd/bd/human_proxied_server.go +++ b/cmd/bd/human_proxied_server.go @@ -122,9 +122,9 @@ func closeHumanProxied(ctx context.Context, id, comment, closeReason, commitVerb if comment != "" { var cerr error if isWisp { - _, cerr = uw.CommentUseCase().AddCommentToWisp(ctx, issue.ID, actor, comment) + _, cerr = uw.CommentUseCase().AddCommentToWisp(ctx, issue.ID, currentActor(), comment) } else { - _, cerr = uw.CommentUseCase().AddCommentToIssue(ctx, issue.ID, actor, comment) + _, cerr = uw.CommentUseCase().AddCommentToIssue(ctx, issue.ID, currentActor(), comment) } if cerr != nil { return humanCloseResult{}, "", fmt.Errorf("adding comment: %w", cerr) @@ -133,9 +133,9 @@ func closeHumanProxied(ctx context.Context, id, comment, closeReason, commitVerb params := domain.CloseIssueParams{Reason: closeReason} if isWisp { - _, err = uw.IssueUseCase().CloseWisp(ctx, issue.ID, params, actor) + _, err = uw.IssueUseCase().CloseWisp(ctx, issue.ID, params, currentActor()) } else { - _, err = uw.IssueUseCase().CloseIssue(ctx, issue.ID, params, actor) + _, err = uw.IssueUseCase().CloseIssue(ctx, issue.ID, params, currentActor()) } if err != nil { return humanCloseResult{}, "", fmt.Errorf("closing bead: %w", err) diff --git a/cmd/bd/jira.go b/cmd/bd/jira.go index 099359b2a..b03574fe9 100644 --- a/cmd/bd/jira.go +++ b/cmd/bd/jira.go @@ -139,7 +139,7 @@ func runJiraSync(cmd *cobra.Command, args []string) error { return HandleErrorRespectJSON("initializing Jira tracker: %v", err) } - engine := tracker.NewEngine(jt, trackerStore, actor) + engine := tracker.NewEngine(jt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } diff --git a/cmd/bd/label.go b/cmd/bd/label.go index 46fcfd177..5f5a768db 100644 --- a/cmd/bd/label.go +++ b/cmd/bd/label.go @@ -165,7 +165,7 @@ func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, ope } } result, uerr := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: issueID, Patch: patch, }) @@ -361,7 +361,7 @@ func removeLabelsByPrefix(ctx context.Context, issueIDs []string, prefix string, outcomes := make([]labelEditOutcome, 0, len(targets)) for _, target := range targets { result, uerr := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: target.issueID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Remove: target.labels}}, }) @@ -707,7 +707,7 @@ var labelPropagateCmd = &cobra.Command{ commitMsg := fmt.Sprintf("bd: propagate label '%s' from %s to %d children", label, parentID, len(children)) err = transactHonoringAutoCommit(ctx, store, commitMsg, func(tx storage.Transaction) error { for _, child := range children { - if err := tx.AddLabel(ctx, child.ID, label, actor); err != nil { + if err := tx.AddLabel(ctx, child.ID, label, currentActor()); err != nil { return fmt.Errorf("add label '%s' on %s: %w", label, child.ID, err) } } @@ -809,7 +809,7 @@ func runLabelRename(ctx context.Context, args []string, dryRun bool) error { if usesProxiedServer() { renamed, merged, err = runLabelRenameProxiedServer(ctx, oldLabel, newLabel) } else { - renamed, merged, _, err = store.RenameLabel(ctx, oldLabel, newLabel, actor) + renamed, merged, _, err = store.RenameLabel(ctx, oldLabel, newLabel, currentActor()) } // Recorded from renamed>0 BEFORE the error check: a rename can commit its // SQL side and still return a non-nil err (e.g. the Dolt publication step diff --git a/cmd/bd/label_proxied_server.go b/cmd/bd/label_proxied_server.go index 9d5b955a1..0bf9215f0 100644 --- a/cmd/bd/label_proxied_server.go +++ b/cmd/bd/label_proxied_server.go @@ -148,7 +148,7 @@ func runLabelRenameProxiedServer(ctx context.Context, oldLabel, newLabel string) err = uow.RunTx(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (string, error) { var rerr error - renamed, merged, _, rerr = uw.LabelUseCase().RenameLabel(ctx, oldLabel, newLabel, actor) //nolint:forbidigo // bulk rename; no role serves it, and Patch.Labels would reintroduce the per-issue add/remove event divergence this delegation replaced + renamed, merged, _, rerr = uw.LabelUseCase().RenameLabel(ctx, oldLabel, newLabel, currentActor()) //nolint:forbidigo // bulk rename; no role serves it, and Patch.Labels would reintroduce the per-issue add/remove event divergence this delegation replaced if rerr != nil { return "", fmt.Errorf("rename label '%s' -> '%s': %w", oldLabel, newLabel, rerr) } @@ -209,9 +209,9 @@ func runLabelPropagateProxiedServer(ctx context.Context, args []string) error { for _, child := range children { var e error if child.Ephemeral { - e = uw.LabelUseCase().AddWispLabel(ctx, child.ID, label, actor) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor + e = uw.LabelUseCase().AddWispLabel(ctx, child.ID, label, currentActor()) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor } else { - e = uw.LabelUseCase().AddLabel(ctx, child.ID, label, actor) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor + e = uw.LabelUseCase().AddLabel(ctx, child.ID, label, currentActor()) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor } if e != nil { return "", fmt.Errorf("add label '%s' on %s: %w", label, child.ID, e) diff --git a/cmd/bd/linear.go b/cmd/bd/linear.go index 8e24cb777..f6ba42990 100644 --- a/cmd/bd/linear.go +++ b/cmd/bd/linear.go @@ -338,14 +338,14 @@ func runLinearSync(cmd *cobra.Command, args []string) error { } } - engine := tracker.NewEngine(lt, trackerStore, actor) + engine := tracker.NewEngine(lt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PullHooks = buildLinearPullHooksForStore(ctx, trackerStore, linearPullHookOptions{ Milestones: milestones, DryRun: dryRun, - Actor: actor, + Actor: currentActor(), }) opts := tracker.SyncOptions{ @@ -488,7 +488,7 @@ func buildLinearPullHooksForStore(ctx context.Context, st tracker.Store, opts li hooks := &tracker.PullHooks{} hookActor := opts.Actor if hookActor == "" { - hookActor = actor + hookActor = currentActor() } var generateID func(context.Context, *types.Issue) error diff --git a/cmd/bd/link.go b/cmd/bd/link.go index 772a88d91..a05be924b 100644 --- a/cmd/bd/link.go +++ b/cmd/bd/link.go @@ -77,13 +77,13 @@ Examples: Type: dt, } - if err := fromStore.AddDependencyWithOptions(ctx, dep, actor, storage.DependencyAddOptions{EmitEvent: true}); err != nil { + if err := fromStore.AddDependencyWithOptions(ctx, dep, currentActor(), storage.DependencyAddOptions{EmitEvent: true}); err != nil { return HandleErrorRespectJSON("%v", err) } warnIfCyclesExist(fromStore) - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "link", IssueIDs: []string{fromID, toID}, }); err != nil { diff --git a/cmd/bd/link_proxied_server.go b/cmd/bd/link_proxied_server.go index aa7fac1b9..aef888968 100644 --- a/cmd/bd/link_proxied_server.go +++ b/cmd/bd/link_proxied_server.go @@ -35,7 +35,7 @@ func runLinkProxiedServer(cmd *cobra.Command, ctx context.Context, args []string // Source-routed, like the direct twin's store.AddDependencyWithOptions: // `bd link` takes whatever id the caller names, and a wisp source has no // row in the issues plane for the edge to hang off. - if _, err := uw.DependencyUseCase().AddDependencies(ctx, []*types.Dependency{dep}, actor, domain.BulkAddDepsOpts{}); err != nil { + if _, err := uw.DependencyUseCase().AddDependencies(ctx, []*types.Dependency{dep}, currentActor(), domain.BulkAddDepsOpts{}); err != nil { return "", err } return fmt.Sprintf("bd: link %s %s", id1, id2), nil diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 9fd38b3b9..901d461ee 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -840,6 +840,10 @@ func refreshBoundCommandConfig(cmd *cobra.Command) { readonlyMode = config.GetBool("readonly") } if !root.PersistentFlags().Changed("actor") { + // Raw write, not setActor: every caller runs in PersistentPreRunE + // BEFORE deferActorGitFallback, which then decides from this value + // whether the git user.name fallback is pending. A call after it would + // need setActor (and the deferral re-run) to keep that decision true. actor = resolveConfiguredActor() } if !root.PersistentFlags().Changed("dolt-auto-commit") { @@ -893,11 +897,74 @@ func resolveConfiguredActor() string { return config.GetString("actor") } +// actorGitFallbackPending records that PersistentPreRunE deferred the +// getActorWithGit fallback (git config user.name, then $USER) instead of +// resolving it eagerly. Most invocations — every read-only list/show/query a +// tool like gascity issues by the hundred — never consult the actor, and the +// eager lookup cost one `git config` subprocess per bd call. currentActor +// performs the deferred resolution on first use, so the resolved value and its +// priority order are unchanged; only the moment of the lookup moves. +// +// Guarded by actorMu together with the resolution it triggers. Plain writes to +// actor elsewhere stay unguarded, exactly as before this was introduced. +var ( + actorMu sync.Mutex + actorGitFallbackPending bool +) + +// deferActorGitFallback replaces the former eager `actor = getActorWithGit()` +// in PersistentPreRunE. A non-empty actor (--actor, BEADS_ACTOR, BD_ACTOR or +// config.yaml) is already the answer getActorWithGit would return, so only an +// empty actor needs the deferred fallback. +func deferActorGitFallback() { + actorMu.Lock() + defer actorMu.Unlock() + actorGitFallbackPending = actor == "" +} + +// currentActor returns the actor for this invocation, resolving a fallback +// deferred by deferActorGitFallback on first use. Command code reads the actor +// through this instead of the raw global so that the git lookup happens only +// when an actor is actually needed. +func currentActor() string { + actorMu.Lock() + defer actorMu.Unlock() + resolvePendingActorLocked() + return actor +} + +// resolvePendingActorLocked performs a deferred fallback. Caller holds actorMu. +func resolvePendingActorLocked() { + if !actorGitFallbackPending { + return + } + actorGitFallbackPending = false + deferred := actor + actor = resolveActorWithGit() + // Keep the CommandContext copy in step, as the eager assignment did + // before syncCommandContext copied it. + if cmdCtx != nil && cmdCtx.Actor == deferred { + cmdCtx.Actor = actor + } +} + // getActorWithGit returns the actor for audit trails with git config fallback. // Priority: --actor flag > BEADS_ACTOR env > BD_ACTOR env (deprecated) > git config user.name > $USER > "unknown" // This provides a sensible default for developers: their git identity is used unless -// explicitly overridden +// explicitly overridden. +// +// After PersistentPreRunE it returns the same value as currentActor (resolving +// a deferred fallback once, into the global, as the eager assignment did). func getActorWithGit() string { + actorMu.Lock() + defer actorMu.Unlock() + resolvePendingActorLocked() + return resolveActorWithGit() +} + +// resolveActorWithGit is getActorWithGit's resolution order, without the +// deferred-fallback bookkeeping. Caller holds actorMu or owns the global. +func resolveActorWithGit() string { // If actor is already set (from --actor flag), use it if actor != "" { return actor @@ -1239,6 +1306,10 @@ var rootCmd = &cobra.Command{ WasSet bool }{dbPath, true} } + // Reads and writes the raw global on purpose: this runs before + // deferActorGitFallback, so no fallback is pending yet and the value + // is only the flag/env/config actor. currentActor() here would resolve + // git user.name eagerly, which is what the deferral exists to avoid. if !cmd.Root().PersistentFlags().Changed("actor") && actor == "" { actor = resolveConfiguredActor() } else if cmd.Root().PersistentFlags().Changed("actor") { @@ -1674,11 +1745,14 @@ var rootCmd = &cobra.Command{ return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapReadonly)) } - // Set actor for audit trail - actor = getActorWithGit() - // Attach actor to the command span now that we have it. - if commandSpan != nil { - commandSpan.SetAttributes(attribute.String("bd.actor", actor)) + // Set actor for audit trail. The git config user.name fallback is + // resolved lazily by currentActor, so commands that never consult the + // actor (read-only queries) do not spawn a git subprocess for it. + deferActorGitFallback() + // Attach actor to the command span. Only a recording span (telemetry + // enabled) needs the value, so a noop span does not force the lookup. + if commandSpan != nil && commandSpan.IsRecording() { + commandSpan.SetAttributes(attribute.String("bd.actor", currentActor())) } // Check if this is a read-only command (GH#804) or an explicitly diff --git a/cmd/bd/markdown.go b/cmd/bd/markdown.go index 2c002ad74..b902ad7a4 100644 --- a/cmd/bd/markdown.go +++ b/cmd/bd/markdown.go @@ -425,8 +425,8 @@ func markdownBatchActor(in createInput) string { if in.createdBy != "" { return in.createdBy } - if actor != "" { - return actor + if currentActor() != "" { + return currentActor() } return "bd" } diff --git a/cmd/bd/merge_slot.go b/cmd/bd/merge_slot.go index 927327472..cfd715394 100644 --- a/cmd/bd/merge_slot.go +++ b/cmd/bd/merge_slot.go @@ -126,7 +126,7 @@ func runMergeSlotCreate(cmd *cobra.Command, args []string) error { } }() - issue, err := store.MergeSlotCreate(rootCtx, actor) + issue, err := store.MergeSlotCreate(rootCtx, currentActor()) if err != nil { return HandleErrorRespectJSON("%v", err) } @@ -222,13 +222,13 @@ func runMergeSlotAcquire(cmd *cobra.Command, args []string) error { holder := mergeSlotHolder if holder == "" { - holder = actor + holder = currentActor() } if holder == "" { return HandleError("no holder specified; use --holder or set BEADS_ACTOR env var") } - result, err := store.MergeSlotAcquire(rootCtx, holder, actor, mergeSlotAddWaiter) + result, err := store.MergeSlotAcquire(rootCtx, holder, currentActor(), mergeSlotAddWaiter) if err != nil { return HandleErrorRespectJSON("%v", err) } @@ -305,7 +305,7 @@ func runMergeSlotRelease(cmd *cobra.Command, args []string) error { } }() - if err := store.MergeSlotRelease(rootCtx, mergeSlotHolder, actor); err != nil { + if err := store.MergeSlotRelease(rootCtx, mergeSlotHolder, currentActor()); err != nil { return HandleErrorRespectJSON("%v", err) } diff --git a/cmd/bd/migrate_issues.go b/cmd/bd/migrate_issues.go index 382d7ac3b..192afcaf9 100644 --- a/cmd/bd/migrate_issues.go +++ b/cmd/bd/migrate_issues.go @@ -600,7 +600,7 @@ func executeMigration(ctx context.Context, s storage.DoltStorage, migrationSet [ for _, id := range migrationSet { if err := tx.UpdateIssue(ctx, id, map[string]interface{}{ "source_repo": to, - }, actor); err != nil { + }, currentActor()); err != nil { return fmt.Errorf("failed to update issue %s: %w", id, err) } } diff --git a/cmd/bd/mol_bond.go b/cmd/bd/mol_bond.go index 3ea600de9..4f8a5f2c2 100644 --- a/cmd/bd/mol_bond.go +++ b/cmd/bd/mol_bond.go @@ -148,21 +148,21 @@ func runMolBond(cmd *cobra.Command, args []string) error { var result *BondResult switch { case aIsProto && bIsProto: - result, err = bondProtoProto(ctx, store, issueA, issueB, in.bondType, in.customTitle, actor) + result, err = bondProtoProto(ctx, store, issueA, issueB, in.bondType, in.customTitle, currentActor()) case aIsProto && !bIsProto: if cookedA { - result, err = bondProtoMolWithSubgraph(ctx, store, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolWithSubgraph(ctx, store, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } else { - result, err = bondProtoMol(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMol(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } case !aIsProto && bIsProto: if cookedB { - result, err = bondProtoMolWithSubgraph(ctx, store, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolWithSubgraph(ctx, store, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } else { - result, err = bondMolProto(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondMolProto(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } default: - result, err = bondMolMol(ctx, store, issueA, issueB, in.bondType, actor) + result, err = bondMolMol(ctx, store, issueA, issueB, in.bondType, currentActor()) } if err != nil { return HandleErrorRespectJSON("bonding: %v", err) diff --git a/cmd/bd/mol_burn.go b/cmd/bd/mol_burn.go index 3062ff794..48293d832 100644 --- a/cmd/bd/mol_burn.go +++ b/cmd/bd/mol_burn.go @@ -190,7 +190,7 @@ func burnMultipleMolecules(ctx context.Context, moleculeIDs []string, dryRun, fo // Batch delete all wisps in one call if len(wispIDs) > 0 { - result, err := burnWisps(ctx, store, wispIDs, actor) + result, err := burnWisps(ctx, store, wispIDs, currentActor()) if err != nil { if !jsonOutput { fmt.Fprintf(os.Stderr, "Error burning wisps: %v\n", err) @@ -296,7 +296,7 @@ func burnWispMolecule(ctx context.Context, resolvedID string, dryRun, force bool } } - result, err := burnWisps(ctx, store, wispIDs, actor) + result, err := burnWisps(ctx, store, wispIDs, currentActor()) if err != nil { return HandleErrorRespectJSON("burning wisp: %v", err) } diff --git a/cmd/bd/mol_current.go b/cmd/bd/mol_current.go index f3d92d33f..6769f01ef 100644 --- a/cmd/bd/mol_current.go +++ b/cmd/bd/mol_current.go @@ -73,7 +73,7 @@ Use --limit or --range to view specific steps: agent := forAgent if agent == "" { - agent = actor + agent = currentActor() } if usesProxiedServer() { diff --git a/cmd/bd/mol_progress.go b/cmd/bd/mol_progress.go index ba35c2b61..df77f9359 100644 --- a/cmd/bd/mol_progress.go +++ b/cmd/bd/mol_progress.go @@ -58,7 +58,7 @@ Example: } moleculeID = resolved } else { - moleculeIDs := findInProgressMoleculeIDs(ctx, store, actor) + moleculeIDs := findInProgressMoleculeIDs(ctx, store, currentActor()) if len(moleculeIDs) == 0 { if jsonOutput { return outputJSON([]interface{}{}) diff --git a/cmd/bd/mol_proxied_server.go b/cmd/bd/mol_proxied_server.go index 6136499dc..04f16955e 100644 --- a/cmd/bd/mol_proxied_server.go +++ b/cmd/bd/mol_proxied_server.go @@ -124,7 +124,7 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { spawnResult, err := cloneSubgraphInto(ctx, w, subgraph, CloneOptions{ Vars: vars, Assignee: in.assignee, - Actor: actor, + Actor: currentActor(), Prefix: types.IDPrefixMol, }) if err != nil { @@ -138,7 +138,7 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { return pourProxiedResult{}, "", fmt.Errorf("loading spawned mol: %w", err) } for _, attach := range attachments { - bondResult, err := bondProtoMolAttachInto(ctx, w, attach.subgraph, attach.issue, spawnedMol, in.attachType, vars, "", actor, false, true) + bondResult, err := bondProtoMolAttachInto(ctx, w, attach.subgraph, attach.issue, spawnedMol, in.attachType, vars, "", currentActor(), false, true) if err != nil { return pourProxiedResult{}, "", fmt.Errorf("attaching %s: %w", attach.id, err) } @@ -278,7 +278,7 @@ func runMolProgressProxiedServer(ctx context.Context, args []string) error { } moleculeID = resolved } else { - moleculeIDs := findInProgressMoleculeIDs(ctx, r, actor) + moleculeIDs := findInProgressMoleculeIDs(ctx, r, currentActor()) if len(moleculeIDs) == 0 { if jsonOutput { return outputJSON([]interface{}{}) @@ -426,13 +426,13 @@ func runMolBondProxiedServer(ctx context.Context, in molBondInput) error { var result *BondResult switch { case aIsProto && bIsProto: - result, err = bondProtoProtoInto(ctx, w, issueA, issueB, in.bondType, in.customTitle, actor) + result, err = bondProtoProtoInto(ctx, w, issueA, issueB, in.bondType, in.customTitle, currentActor()) case aIsProto && !bIsProto: - result, err = bondProtoMolAttachInto(ctx, w, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolAttachInto(ctx, w, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) case !aIsProto && bIsProto: - result, err = bondProtoMolAttachInto(ctx, w, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolAttachInto(ctx, w, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) default: - result, err = bondMolMolInto(ctx, w, issueA, issueB, in.bondType, actor) + result, err = bondMolMolInto(ctx, w, issueA, issueB, in.bondType, currentActor()) } if err != nil { return bondTxResult{}, "", fmt.Errorf("bonding: %w", err) @@ -501,7 +501,7 @@ func runMolSquashProxiedServer(ctx context.Context, in molSquashInput) error { return &SquashResult{MoleculeID: moleculeID, SquashedCount: 0}, "", nil } - result, err := squashMoleculeInto(ctx, w, subgraph.Root, wispChildren, in.keepChildren, in.summary, actor) + result, err := squashMoleculeInto(ctx, w, subgraph.Root, wispChildren, in.keepChildren, in.summary, currentActor()) if err != nil { return nil, "", fmt.Errorf("squashing molecule: %w", err) } @@ -638,7 +638,7 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b if len(wispIDs) > 0 { // The wisp half's direct twin is burnWisps -> `transact`. result, err := uow.RunTxResult(explicitCommitPointContext(ctx), uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*BurnResult, string, error) { - r, err := burnWispsInto(ctx, newUOWMolWriter(uw), wispIDs, actor) + r, err := burnWispsInto(ctx, newUOWMolWriter(uw), wispIDs, currentActor()) if err != nil { return nil, "", err } @@ -672,7 +672,7 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b IDs: issueIDs, Cascade: true, UpdateTextReferences: true, - }, actor) + }, currentActor()) if err != nil { return BurnResult{}, "", fmt.Errorf("burning %s: %w", id, err) } diff --git a/cmd/bd/mol_squash.go b/cmd/bd/mol_squash.go index 223e31955..39363cbdd 100644 --- a/cmd/bd/mol_squash.go +++ b/cmd/bd/mol_squash.go @@ -178,7 +178,7 @@ func runMolSquash(cmd *cobra.Command, args []string) error { return nil } - result, err := squashMolecule(ctx, store, subgraph.Root, wispChildren, in.keepChildren, in.summary, actor) + result, err := squashMolecule(ctx, store, subgraph.Root, wispChildren, in.keepChildren, in.summary, currentActor()) if err != nil { return HandleErrorRespectJSON("squashing molecule: %v", err) } diff --git a/cmd/bd/mutate_proxied_server.go b/cmd/bd/mutate_proxied_server.go index 7008e9a41..fa72c238e 100644 --- a/cmd/bd/mutate_proxied_server.go +++ b/cmd/bd/mutate_proxied_server.go @@ -54,15 +54,15 @@ func proxiedUpdateIssueFields(ctx context.Context, id, commitMsg string, updates // bd-98s5c: an unguarded assignee update (bd assign via the proxied // server) must not silently overwrite another actor's live claim. if newAssignee, ok := updates["assignee"].(string); ok { - if err := validateIssueReassignable(id, issue, actor, newAssignee, + if err := validateIssueReassignable(id, issue, currentActor(), newAssignee, uowClaimPoolAliases(ctx, uw), force); err != nil { return err } } if isWisp { - return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, currentActor()) } - return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, currentActor()) }) } @@ -95,7 +95,7 @@ func proxiedAssign(ctx context.Context, id, assignee string, force bool, ifRevis // from the guarded write below instead of this policy refusal — see // ifRevisionAlreadyStale's doc. if !ifRevisionAlreadyStale(current, ifRevision) { - if verr := validateIssueReassignable(id, current, actor, assignee, + if verr := validateIssueReassignable(id, current, currentActor(), assignee, uowClaimPoolAliases(ctx, uw), force); verr != nil { return struct{}{}, verr } @@ -111,7 +111,7 @@ func proxiedAssign(ctx context.Context, id, assignee string, force bool, ifRevis return nil, err } result, err := runCommandUpdateMutation(ctx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: id, patch: issueops.IssuePatch{ Assignee: issueops.Field[string]{Set: true, Value: assignee}, @@ -183,9 +183,9 @@ func runNoteProxiedServer(ctx context.Context, id, noteText string) error { combined += noteText updates := map[string]any{"notes": combined} if isWisp { - return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, currentActor()) } - return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, currentActor()) }) if err != nil { return HandleErrorRespectJSON("note %s: %v", id, err) @@ -245,7 +245,7 @@ func runTagProxiedServer(ctx context.Context, id, label string) error { return HandleErrorRespectJSON("tag %s: %v", id, err) } result, err := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: details.ID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Add: []string{label}}}, }) diff --git a/cmd/bd/note.go b/cmd/bd/note.go index b0c8eeeab..9ee62cb9e 100644 --- a/cmd/bd/note.go +++ b/cmd/bd/note.go @@ -173,10 +173,10 @@ To read notes on an issue, use: bd show `, updates := map[string]interface{}{ issueops.OpAppendNotes: noteText, } - if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, actor); err != nil { + if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, currentActor()); err != nil { return HandleErrorRespectJSON("updating %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "note", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/notion.go b/cmd/bd/notion.go index 664cd31f6..3ec42b336 100644 --- a/cmd/bd/notion.go +++ b/cmd/bd/notion.go @@ -467,7 +467,7 @@ func runNotionSync(cmd *cobra.Command, _ []string) error { return HandleError("initializing Notion tracker: %v", err) } - engine := tracker.NewEngine(nt, store, actor) + engine := tracker.NewEngine(nt, store, currentActor()) engine.PullHooks = buildNotionPullHooks(ctx) unsupportedStats := newNotionUnsupportedPushStats() engine.PushHooks = buildNotionPushHooks(ctx, nt, unsupportedStats) diff --git a/cmd/bd/pour.go b/cmd/bd/pour.go index 91aa946eb..7260365ee 100644 --- a/cmd/bd/pour.go +++ b/cmd/bd/pour.go @@ -198,7 +198,7 @@ func runPour(cmd *cobra.Command, args []string) error { return nil } - result, err := spawnMolecule(ctx, store, subgraph, vars, in.assignee, actor, false, types.IDPrefixMol) + result, err := spawnMolecule(ctx, store, subgraph, vars, in.assignee, currentActor(), false, types.IDPrefixMol) if err != nil { return HandleError("pouring proto: %v", err) } @@ -211,7 +211,7 @@ func runPour(cmd *cobra.Command, args []string) error { } for _, attach := range attachments { - bondResult, err := bondProtoMol(ctx, store, attach.issue, spawnedMol, in.attachType, vars, "", actor, false, true) + bondResult, err := bondProtoMol(ctx, store, attach.issue, spawnedMol, in.attachType, vars, "", currentActor(), false, true) if err != nil { return HandleError("attaching %s: %v", attach.id, err) } diff --git a/cmd/bd/prime.go b/cmd/bd/prime.go index adec80396..f314fcb4a 100644 --- a/cmd/bd/prime.go +++ b/cmd/bd/prime.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads" internalbeads "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/memoryops" ) @@ -303,10 +304,16 @@ func primeWorkspaceDir() string { // failure falls back to a cwd probe instead of propagating — so this no // longer returns an error. If a future caller needs one, reintroduce it then. func primeGitCmd(ctx context.Context, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Dir = primeGitDir() + return cmd +} + +// primeGitDir is the directory primeGitCmd runs git in ("" for the process +// cwd): the -C target, else the cwd repository root from the RepoContext. +func primeGitDir() string { if ws := primeWorkspaceDir(); ws != "" { - cmd := exec.CommandContext(ctx, "git", args...) - cmd.Dir = ws - return cmd + return ws } rc, err := internalbeads.GetRepoContext() if err != nil { @@ -317,9 +324,9 @@ func primeGitCmd(ctx context.Context, args ...string) *exec.Cmd { // workspace, so probe it directly instead of giving up. The SEC-003 // boundary on BEADS_DIR stays enforced by the callers that consume // BEADS_DIR itself; these probes only ask git about its own workspace. - return exec.CommandContext(ctx, "git", args...) + return "" } - return rc.GitCmdCWD(ctx, args...) + return rc.CWDRepoRoot // GitCmdCWD's directory; "" leaves the process cwd } // outputHookJSON wraps content in the SessionStart hook JSON envelope shared @@ -422,8 +429,18 @@ var primeAgentProfile = func() config.AgentProfile { // `git remote` in the workspace succeeds. primeGitCmd supplies that fallback, // which keeps the -C behavior above intact. The SEC-003 boundary on BEADS_DIR // remains enforced elsewhere. +// +// The probe runs on every command in an embedded workspace whose +// backup.enabled is unset (isBackupAutoEnabled), so it is answered from the +// config files when git.HasRemoteInProcess can prove the answer, and by +// `git remote` otherwise. var primeHasGitRemote = func() bool { - cmd := primeGitCmd(context.Background(), "remote") + dir := primeGitDir() + if has, ok := git.HasRemoteInProcess(dir); ok { + return has + } + cmd := exec.CommandContext(context.Background(), "git", "remote") + cmd.Dir = dir out, err := cmd.Output() if err != nil { return false diff --git a/cmd/bd/priority.go b/cmd/bd/priority.go index 481afecec..66a633e71 100644 --- a/cmd/bd/priority.go +++ b/cmd/bd/priority.go @@ -78,10 +78,10 @@ Examples: updates := map[string]interface{}{ "priority": priority, } - if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, actor); err != nil { + if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, currentActor()); err != nil { return HandleErrorRespectJSON("updating %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "priority", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/promote.go b/cmd/bd/promote.go index b1238abbf..44a42eea0 100644 --- a/cmd/bd/promote.go +++ b/cmd/bd/promote.go @@ -97,13 +97,13 @@ Examples: return HandleErrorRespectJSON("%s is not a wisp (already persistent)", fullID) } - if err := store.PromoteFromEphemeral(ctx, fullID, actor); err != nil { + if err := store.PromoteFromEphemeral(ctx, fullID, currentActor()); err != nil { return HandleErrorRespectJSON("promoting %s: %v", fullID, err) } // Add promotion comment (issue is now in permanent table, AddComment routes correctly // via GetIssue fallback) - if err := store.AddComment(ctx, fullID, actor, promotionComment(reason)); err != nil { + if err := store.AddComment(ctx, fullID, currentActor(), promotionComment(reason)); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to add promotion comment to %s: %v\n", fullID, err) } diff --git a/cmd/bd/promote_proxied_server.go b/cmd/bd/promote_proxied_server.go index edadce2d5..38f6fb547 100644 --- a/cmd/bd/promote_proxied_server.go +++ b/cmd/bd/promote_proxied_server.go @@ -62,10 +62,10 @@ func runPromoteProxiedServer(ctx context.Context, id, reason string) error { } err = uow.RunTx(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (string, error) { - if err := uw.IssueUseCase().PromoteWisp(ctx, fullID, actor); err != nil { + if err := uw.IssueUseCase().PromoteWisp(ctx, fullID, currentActor()); err != nil { return "", err } - if _, err := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, actor, promotionComment(reason)); err != nil { + if _, err := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, currentActor(), promotionComment(reason)); err != nil { return "", fmt.Errorf("adding promotion comment: %w", err) } return fmt.Sprintf("bd: promote %s", fullID), nil diff --git a/cmd/bd/purge.go b/cmd/bd/purge.go index 0337db793..cead0ae98 100644 --- a/cmd/bd/purge.go +++ b/cmd/bd/purge.go @@ -165,7 +165,7 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { } request := issueops.SweepRequest{ - Actor: actor, + Actor: currentActor(), Tier: scope.tier, IDPattern: pattern, ProtectReferenced: scope.protectReferenced, diff --git a/cmd/bd/quick.go b/cmd/bd/quick.go index a0cdbf550..e9ab99c03 100644 --- a/cmd/bd/quick.go +++ b/cmd/bd/quick.go @@ -94,7 +94,7 @@ Example: // The issue and its parent-child edge commit in one transaction; a // failed edge rolls back the create instead of leaving a dep-less // child behind (same contract as bd create). - if err := createIssueWithDeps(ctx, store, issue, actor, createDepEdges{parentID: parentID}); err != nil { + if err := createIssueWithDeps(ctx, store, issue, currentActor(), createDepEdges{parentID: parentID}); err != nil { return HandleError("%v", err) } diff --git a/cmd/bd/quick_proxied_server.go b/cmd/bd/quick_proxied_server.go index 3035863bb..46ab79c62 100644 --- a/cmd/bd/quick_proxied_server.go +++ b/cmd/bd/quick_proxied_server.go @@ -47,7 +47,7 @@ func runQuickProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin Labels: labels, InheritLabelsFromParent: parentID != "", } - result, err := uw.IssueUseCase().CreateIssue(ctx, params, actor) + result, err := uw.IssueUseCase().CreateIssue(ctx, params, currentActor()) if err != nil { return nil, "", err } diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index babcefa35..7e7e8aacf 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -159,7 +159,7 @@ This is useful for agents executing molecules to see which steps can run next.`, return nil } claimed := res.Claimed - if err := commitPendingIfEmbedded(ctx, activeStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, activeStore, currentActor(), doltAutoCommitParams{ Command: "ready", IssueIDs: []string{claimed.ID}, }); err != nil { diff --git a/cmd/bd/ready_input.go b/cmd/bd/ready_input.go index 0643ae3bd..e55a830c4 100644 --- a/cmd/bd/ready_input.go +++ b/cmd/bd/ready_input.go @@ -260,7 +260,7 @@ func readyRoleRequest(in readyInput) issueops.ReadyRequest { // claimNextRequest is `bd ready --claim`'s request to the ReadyClaimer role: // the shared ready question above, plus the claimant. func claimNextRequest(in readyInput) issueops.ClaimNextRequest { - return issueops.ClaimNextRequest{Actor: actor, Filter: readyRoleRequest(in)} + return issueops.ClaimNextRequest{Actor: currentActor(), Filter: readyRoleRequest(in)} } // briefModeConflict reports the usage error for a --brief combination that no diff --git a/cmd/bd/reclaim.go b/cmd/bd/reclaim.go index 34b40d98b..2a5262d99 100644 --- a/cmd/bd/reclaim.go +++ b/cmd/bd/reclaim.go @@ -104,7 +104,7 @@ Examples: } ctx := rootCtx - reclaimed, err := store.ReclaimExpiredLeases(ctx, olderThan, filter, actor) + reclaimed, err := store.ReclaimExpiredLeases(ctx, olderThan, filter, currentActor()) if err != nil { return HandleErrorRespectJSON("reclaim: %v", err) } @@ -113,7 +113,7 @@ Examples: for _, r := range reclaimed { ids = append(ids, r.ID) } - if err := commitPendingIfEmbedded(ctx, store, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, store, currentActor(), doltAutoCommitParams{ Command: "reclaim", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/relate.go b/cmd/bd/relate.go index 987ccdff9..f87f644d4 100644 --- a/cmd/bd/relate.go +++ b/cmd/bd/relate.go @@ -111,7 +111,7 @@ func runRelate(cmd *cobra.Command, args []string) error { } // bd relate is an explicit dependency verb, so it records history like // bd link / bd dep add (EmitEvent); only structural edge wiring stays silent. - if err := store.AddDependencyWithOptions(ctx, dep1, actor, storage.DependencyAddOptions{EmitEvent: true}); err != nil { + if err := store.AddDependencyWithOptions(ctx, dep1, currentActor(), storage.DependencyAddOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to add relates-to %s -> %s: %w", id1, id2, err) } // Add id2 -> id1 (bidirectional) @@ -120,7 +120,7 @@ func runRelate(cmd *cobra.Command, args []string) error { DependsOnID: id1, Type: types.DepRelatesTo, } - if err := store.AddDependencyWithOptions(ctx, dep2, actor, storage.DependencyAddOptions{EmitEvent: true}); err != nil { + if err := store.AddDependencyWithOptions(ctx, dep2, currentActor(), storage.DependencyAddOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to add relates-to %s -> %s: %w", id2, id1, err) } @@ -190,11 +190,11 @@ func runUnrelate(cmd *cobra.Command, args []string) error { // bd unrelate is an explicit dependency verb, so it records history // (EmitEvent) like bd dep remove; only structural teardown stays silent. // Remove id1 -> id2 - if err := store.RemoveDependencyWithOptions(ctx, id1, id2, actor, storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { + if err := store.RemoveDependencyWithOptions(ctx, id1, id2, currentActor(), storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to remove relates-to %s -> %s: %w", id1, id2, err) } // Remove id2 -> id1 (bidirectional) - if err := store.RemoveDependencyWithOptions(ctx, id2, id1, actor, storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { + if err := store.RemoveDependencyWithOptions(ctx, id2, id1, currentActor(), storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to remove relates-to %s -> %s: %w", id2, id1, err) } diff --git a/cmd/bd/relate_proxied_server.go b/cmd/bd/relate_proxied_server.go index 601b1a30d..34a6d688f 100644 --- a/cmd/bd/relate_proxied_server.go +++ b/cmd/bd/relate_proxied_server.go @@ -66,7 +66,7 @@ func runRelateProxiedServer(ctx context.Context, args []string) error { } // bd relate is an explicit dependency verb, so the bulk path's // EmitEvent trail matches the direct route's history behavior. - if _, err := uw.DependencyUseCase().AddDependencies(ctx, deps, actor, domain.BulkAddDepsOpts{}); err != nil { + if _, err := uw.DependencyUseCase().AddDependencies(ctx, deps, currentActor(), domain.BulkAddDepsOpts{}); err != nil { return "", err } return fmt.Sprintf("bd: relate %s %s", id1, id2), nil @@ -111,11 +111,11 @@ func runUnrelateProxiedServer(ctx context.Context, args []string) error { return "", fmt.Errorf("issue not found: %s", id) } } - removed1, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id1, id2, actor) + removed1, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id1, id2, currentActor()) if err != nil { return "", fmt.Errorf("failed to remove relates-to %s -> %s: %w", id1, id2, err) } - removed2, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id2, id1, actor) + removed2, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id2, id1, currentActor()) if err != nil { return "", fmt.Errorf("failed to remove relates-to %s -> %s: %w", id2, id1, err) } diff --git a/cmd/bd/rename_prefix.go b/cmd/bd/rename_prefix.go index 0ff1c8c91..68936732b 100644 --- a/cmd/bd/rename_prefix.go +++ b/cmd/bd/rename_prefix.go @@ -112,7 +112,7 @@ NOTE: This is a rare operation. Most users never need this command.`, ) } - if err := repairPrefixes(ctx, store, actor, newPrefix, issues, prefixes, dryRun); err != nil { + if err := repairPrefixes(ctx, store, currentActor(), newPrefix, issues, prefixes, dryRun); err != nil { return HandleError("failed to repair prefixes: %v", err) } if !dryRun { @@ -456,7 +456,7 @@ func renamePrefixInDB(ctx context.Context, oldPrefix, newPrefix string, issues [ continue } - if err := renameIssueKeepingBeadGates(ctx, store, issue, newID, gates[oldID], actor); err != nil { + if err := renameIssueKeepingBeadGates(ctx, store, issue, newID, gates[oldID], currentActor()); err != nil { return fmt.Errorf("failed to update issue %s: %w", oldID, err) } } diff --git a/cmd/bd/reopen.go b/cmd/bd/reopen.go index 6bdb3f7cd..6a6635e23 100644 --- a/cmd/bd/reopen.go +++ b/cmd/bd/reopen.go @@ -96,7 +96,7 @@ another actor won the race, so retrying the same guard is pointless).`, continue } reopened, err := ops.Reopen(opsCtx, issueops.ReopenRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fullID, Reason: reason, ExpectedVersion: ifRevision, @@ -150,7 +150,7 @@ another actor won the race, so retrying the same guard is pointless).`, } for s, ids := range mutatedStores { - if err := commitPendingIfEmbedded(ctx, s, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, s, currentActor(), doltAutoCommitParams{ Command: "reopen", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/reopen_proxied_server.go b/cmd/bd/reopen_proxied_server.go index a9fadd638..e61f85a22 100644 --- a/cmd/bd/reopen_proxied_server.go +++ b/cmd/bd/reopen_proxied_server.go @@ -66,7 +66,7 @@ func runReopenProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri reopenedIssues := []*types.Issue{} for _, target := range targets { result, err := lifecycle.Reopen(ctx, issueops.ReopenRequest{ - Actor: actor, + Actor: currentActor(), IssueID: target.id, Reason: reason, ExpectedVersion: ifRevision, @@ -94,7 +94,7 @@ func runReopenProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri continue } - audit.LogFieldChange(target.id, "status", string(target.status), string(types.StatusOpen), actor, reason) + audit.LogFieldChange(target.id, "status", string(target.status), string(types.StatusOpen), currentActor(), reason) if jsonOut { if issue := result.Issue; issue != nil { // `bd reopen` has never printed dependency records, on either diff --git a/cmd/bd/ship.go b/cmd/bd/ship.go index a8c0908ed..9e56be5bd 100644 --- a/cmd/bd/ship.go +++ b/cmd/bd/ship.go @@ -127,7 +127,7 @@ func runShip(cmd *cobra.Command, args []string) error { return nil } - if err := store.AddLabel(ctx, issue.ID, providesLabel, actor); err != nil { + if err := store.AddLabel(ctx, issue.ID, providesLabel, currentActor()); err != nil { return HandleErrorRespectJSON("adding label: %v", err) } diff --git a/cmd/bd/startup_git_exec_embedded_test.go b/cmd/bd/startup_git_exec_embedded_test.go new file mode 100644 index 000000000..a74194207 --- /dev/null +++ b/cmd/bd/startup_git_exec_embedded_test.go @@ -0,0 +1,176 @@ +//go:build cgo && linux + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// gitExecRecorder puts a `git` shim first on PATH that logs each invocation's +// arguments and then runs the real git, so a test can count the git processes +// a bd subprocess spawns. +type gitExecRecorder struct { + shimDir, log string +} + +func newGitExecRecorder(t *testing.T) *gitExecRecorder { + t.Helper() + realGit, err := exec.LookPath("git") + if err != nil { + t.Skipf("git not found: %v", err) + } + r := &gitExecRecorder{shimDir: t.TempDir()} + r.log = filepath.Join(r.shimDir, "git-calls.log") + script := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> '" + r.log + "'\nexec '" + realGit + "' \"$@\"\n" + if err := os.WriteFile(filepath.Join(r.shimDir, "git"), []byte(script), 0o755); err != nil { // #nosec G306 -- test shim must be executable + t.Fatal(err) + } + return r +} + +// run runs bd in dir with the shim on PATH and returns stdout plus the git +// invocations it made. +func (r *gitExecRecorder) run(t *testing.T, bd, dir string, args ...string) (string, []string) { + t.Helper() + if err := os.Remove(r.log); err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + cmd := exec.Command(bd, args...) + cmd.Dir = dir + // Drop every GIT_* variable but the config-file selectors: any of + // internal/git's discovery overrides (GIT_DIR, GIT_WORK_TREE, + // GIT_COMMON_DIR, GIT_CEILING_DIRECTORIES, GIT_DISCOVERY_ACROSS_FILESYSTEM, + // GIT_OBJECT_DIRECTORY, GIT_CONFIG_PARAMETERS, GIT_CONFIG_COUNT with its + // GIT_CONFIG_KEY_n/VALUE_n, ...) exported by the runner — a `git -c` + // wrapper, a hook context — would rightly send bd back to git rev-parse. + var env []string + for _, kv := range envWithout(envWithout(bdEnv(dir), "BD_ACTOR"), "PATH") { + name, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(name, "GIT_") && name != "GIT_CONFIG_GLOBAL" && + name != "GIT_CONFIG_SYSTEM" && name != "GIT_CONFIG_NOSYSTEM" { + continue + } + env = append(env, kv) + } + // backup.enabled stays at its default, so embedded auto-backup asks + // whether a git remote exists after every command (isBackupAutoEnabled). + // Global and system git config are pinned (empty, none) so the answer + // does not depend on the host's ~/.gitconfig — an include there would + // rightly send that probe to `git remote`. + env = envWithout(envWithout(env, "BD_BACKUP_ENABLED"), "BEADS_BACKUP_ENABLED") + for _, name := range []string{"GIT_CONFIG_GLOBAL", "GIT_CONFIG_SYSTEM", "GIT_CONFIG_NOSYSTEM"} { + env = envWithout(env, name) + } + emptyGlobal := filepath.Join(r.shimDir, "empty.gitconfig") + if err := os.WriteFile(emptyGlobal, nil, 0o600); err != nil { + t.Fatal(err) + } + cmd.Env = append(env, "GIT_CONFIG_GLOBAL="+emptyGlobal, "GIT_CONFIG_NOSYSTEM=1", + "PATH="+r.shimDir+string(os.PathListSeparator)+os.Getenv("PATH")) + out, err := cmd.Output() + if err != nil { + var stderr string + if exit, ok := err.(*exec.ExitError); ok { + stderr = string(exit.Stderr) + } + t.Fatalf("bd %s: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), err, out, stderr) + } + data, err := os.ReadFile(r.log) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + var calls []string + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line != "" { + calls = append(calls, line) + } + } + return string(out), calls +} + +// TestEmbeddedReadOnlyCommandsSpawnNoGit pins the per-call cost tools that +// call bd by the hundred depend on: a read-only command resolves the +// repository in-process, never needs the actor, and decides embedded +// auto-backup (default backup.enabled, with and without a git remote) without +// git, so it runs no git at all — from the main checkout and from a linked +// worktree sharing its database. A write still resolves the actor from git +// config user.name, lazily and to the same value. +func TestEmbeddedReadOnlyCommandsSpawnNoGit(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "gx") + issue := bdCreate(t, bd, dir, "startup git probe") + + worktree := filepath.Join(filepath.Dir(dir), filepath.Base(dir)+"-wt") + t.Cleanup(func() { _ = os.RemoveAll(worktree) }) + noHooks := "core.hooksPath=" + t.TempDir() // keep bd's installed hooks out of fixture setup + for _, args := range [][]string{ + {"-c", noHooks, "commit", "--allow-empty", "--no-verify", "-m", "base"}, + {"-c", noHooks, "worktree", "add", "-q", worktree}, + } { + cmd := gitCommand(args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + } + + rec := newGitExecRecorder(t) + reads := func(phase string) { + t.Helper() + for _, location := range []struct{ name, dir string }{ + {"main checkout", dir}, + {"linked worktree", worktree}, + } { + for _, args := range [][]string{ + {"list", "--json"}, + {"show", issue.ID, "--json"}, + } { + out, calls := rec.run(t, bd, location.dir, args...) + if !strings.Contains(out, issue.ID) { + t.Fatalf("%s, %s: bd %s did not find %s:\n%s", phase, location.name, args[0], issue.ID, out) + } + if len(calls) != 0 { + t.Errorf("%s, %s: bd %s spawned %d git process(es), want 0:\n%s", + phase, location.name, args[0], len(calls), strings.Join(calls, "\n")) + } + } + } + } + // No git remote: auto-backup stays off, decided without running git. + reads("no git remote") + // With a remote, auto-backup turns on (still decided in-process); the + // first read may run the backup itself, which needs no git either. + addRemote := gitCommand("remote", "add", "origin", "https://example.invalid/gitcalls.git") + addRemote.Dir, addRemote.Env = dir, bdEnv(dir) + if out, err := addRemote.CombinedOutput(); err != nil { + t.Fatalf("git remote add: %v\n%s", err, out) + } + reads("git remote configured") + + // Positive control for the shim, and the deferred actor's value: a write + // asks git for user.name exactly once and records it as before. + out, calls := rec.run(t, bd, dir, "create", "--json", "lazy actor") + created := parseIssueJSON(t, []byte(out)) + if created.CreatedBy != "Test" { + t.Errorf("created_by = %q, want git user.name %q", created.CreatedBy, "Test") + } + nameLookups := 0 + for _, call := range calls { + if call == "config user.name" { + nameLookups++ + } + } + if nameLookups != 1 { + t.Errorf("bd create ran `git config user.name` %d times, want 1; calls:\n%s", nameLookups, strings.Join(calls, "\n")) + } +} diff --git a/cmd/bd/state.go b/cmd/bd/state.go index a3ad36623..b689b0f28 100644 --- a/cmd/bd/state.go +++ b/cmd/bd/state.go @@ -226,7 +226,7 @@ func runSetState(ctx context.Context, issueID, dimension, newValue, reason strin // could not write, so an event that disagreed with its subject's plane // fails the command instead of silently landing in the wrong table. created, err := lifecycle.Create(ctx, issueops.CreateRequest{ - Actor: actor, + Actor: currentActor(), Issue: &types.Issue{ Title: eventTitle, Description: eventDesc, @@ -254,7 +254,7 @@ func runSetState(ctx context.Context, issueID, dimension, newValue, reason strin // failed. The plane needs no branch: IssuePlaneOnly stays false and the // role resolves it inside its own transaction. if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fullID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{ Add: []string{newLabel}, diff --git a/cmd/bd/status.go b/cmd/bd/status.go index 855f67679..f5deaac38 100644 --- a/cmd/bd/status.go +++ b/cmd/bd/status.go @@ -88,7 +88,7 @@ Examples: // --no-blocked is not consulted here, and never was: an // assignee-scoped summary computes both numbers by a route that has // no fast path (issueops.StatsReporter.AssigneeStats). - result, err = reporter.AssigneeStats(rootCtx, issueops.AssigneeStatsRequest{Assignee: actor}) + result, err = reporter.AssigneeStats(rootCtx, issueops.AssigneeStatsRequest{Assignee: currentActor()}) } else { result, err = reporter.Stats(rootCtx, issueops.StatsRequest{SkipBlocked: noBlocked}) if err == nil && noBlocked && result.Summary.BlockedIssues != nil { diff --git a/cmd/bd/swarm.go b/cmd/bd/swarm.go index 996eefc40..9d066a023 100644 --- a/cmd/bd/swarm.go +++ b/cmd/bd/swarm.go @@ -1001,10 +1001,10 @@ Examples: Status: types.StatusOpen, Priority: issue.Priority, IssueType: types.TypeEpic, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.CreateIssue(ctx, wrapperEpic, actor); err != nil { + if err := store.CreateIssue(ctx, wrapperEpic, currentActor()); err != nil { return HandleErrorRespectJSON("failed to create wrapper epic: %v", err) } @@ -1012,9 +1012,9 @@ Examples: IssueID: issue.ID, DependsOnID: wrapperEpic.ID, Type: types.DepParentChild, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.AddDependency(ctx, dep, actor); err != nil { + if err := store.AddDependency(ctx, dep, currentActor()); err != nil { return HandleErrorRespectJSON("failed to link issue to epic: %v", err) } @@ -1081,10 +1081,10 @@ Examples: IssueType: "molecule", MolType: types.MolTypeSwarm, Assignee: coordinator, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.CreateIssue(ctx, swarmMol, actor); err != nil { + if err := store.CreateIssue(ctx, swarmMol, currentActor()); err != nil { return HandleErrorRespectJSON("failed to create swarm molecule: %v", err) } @@ -1092,9 +1092,9 @@ Examples: IssueID: swarmMol.ID, DependsOnID: epicID, Type: types.DepRelatesTo, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.AddDependency(ctx, dep, actor); err != nil { + if err := store.AddDependency(ctx, dep, currentActor()); err != nil { return HandleErrorRespectJSON("failed to link swarm to epic: %v", err) } diff --git a/cmd/bd/sync_push_pull.go b/cmd/bd/sync_push_pull.go index 63a2263ff..0e6b289da 100644 --- a/cmd/bd/sync_push_pull.go +++ b/cmd/bd/sync_push_pull.go @@ -272,7 +272,7 @@ func runADOPush(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Azure DevOps tracker: %w", err) } - engine := tracker.NewEngine(at, store, actor) + engine := tracker.NewEngine(at, store, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -319,7 +319,7 @@ func runADOPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Azure DevOps tracker: %w", err) } - engine := tracker.NewEngine(at, store, actor) + engine := tracker.NewEngine(at, store, currentActor()) engine.PullHooks = buildADOPullHooks(ctx, at, false, false, new(int), engine.OnWarning) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -369,7 +369,7 @@ func runJiraPush(cmd *cobra.Command, args []string) error { return HandleError("initializing Jira tracker: %v", err) } - engine := tracker.NewEngine(jt, trackerStore, actor) + engine := tracker.NewEngine(jt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PushHooks = buildJiraPushHooksForStore(ctx, trackerStore) @@ -417,7 +417,7 @@ func runJiraPull(cmd *cobra.Command, args []string) error { return HandleError("initializing Jira tracker: %v", err) } - engine := tracker.NewEngine(jt, trackerStore, actor) + engine := tracker.NewEngine(jt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -487,7 +487,7 @@ func runLinearPush(cmd *cobra.Command, args []string) error { return HandleError("%v", err) } - engine := tracker.NewEngine(lt, trackerStore, actor) + engine := tracker.NewEngine(lt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PushHooks = buildLinearPushHooksForStore(ctx, trackerStore, lt, len(args) > 0) @@ -552,12 +552,12 @@ func runLinearPull(cmd *cobra.Command, args []string) error { return HandleError("initializing Linear tracker: %v", err) } - engine := tracker.NewEngine(lt, trackerStore, actor) + engine := tracker.NewEngine(lt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PullHooks = buildLinearPullHooksForStore(ctx, trackerStore, linearPullHookOptions{ DryRun: dryRun, - Actor: actor, + Actor: currentActor(), }) result, err := engine.Sync(ctx, tracker.SyncOptions{ @@ -606,7 +606,7 @@ func runGitHubPush(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing GitHub tracker: %w", err) } - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PushHooks = buildGitHubPushHooks(gt) @@ -668,7 +668,7 @@ func runGitHubPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing GitHub tracker: %w", err) } - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.PullHooks = buildGitHubPullHooks(ctx) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -719,7 +719,7 @@ func runGitLabPush(cmd *cobra.Command, args []string) error { } out := cmd.OutOrStdout() - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.OnMessage = func(msg string) { if !jsonOutput { fmt.Fprintln(out, " "+msg) @@ -821,7 +821,7 @@ func runGitLabPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing GitLab tracker: %w", err) } - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.PullHooks = buildGitLabPullHooks(ctx) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -875,7 +875,7 @@ func runNotionPush(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Notion tracker: %w", err) } - engine := tracker.NewEngine(nt, store, actor) + engine := tracker.NewEngine(nt, store, currentActor()) unsupportedStats := newNotionUnsupportedPushStats() engine.PushHooks = buildNotionPushHooks(ctx, nt, unsupportedStats) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } @@ -929,7 +929,7 @@ func runNotionPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Notion tracker: %w", err) } - engine := tracker.NewEngine(nt, store, actor) + engine := tracker.NewEngine(nt, store, currentActor()) engine.PullHooks = buildNotionPullHooks(ctx) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } diff --git a/cmd/bd/tag.go b/cmd/bd/tag.go index 096b5f66c..02e4db5de 100644 --- a/cmd/bd/tag.go +++ b/cmd/bd/tag.go @@ -67,10 +67,10 @@ Examples: return HandleErrorRespectJSON("%s", err) } - if err := issueStore.AddLabel(ctx, result.ResolvedID, label, actor); err != nil { + if err := issueStore.AddLabel(ctx, result.ResolvedID, label, currentActor()); err != nil { return HandleErrorRespectJSON("adding label to %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "tag", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/unclaim.go b/cmd/bd/unclaim.go index f900e1750..3dd59a8fe 100644 --- a/cmd/bd/unclaim.go +++ b/cmd/bd/unclaim.go @@ -90,9 +90,9 @@ Examples: var unclaimErr error if conditional { - unclaimErr = issueStore.UnclaimIssueIfAssignee(ctx, fullID, actor, ifAssignee) + unclaimErr = issueStore.UnclaimIssueIfAssignee(ctx, fullID, currentActor(), ifAssignee) } else { - unclaimErr = issueStore.UnclaimIssue(ctx, fullID, actor, force) + unclaimErr = issueStore.UnclaimIssue(ctx, fullID, currentActor(), force) } if unclaimErr != nil { fmt.Fprintf(os.Stderr, "Error unclaiming %s: %v\n", fullID, unclaimErr) @@ -102,7 +102,7 @@ Examples: } if reason != "" { - if _, err := issueStore.AddIssueComment(ctx, fullID, actor, reason); err != nil { + if _, err := issueStore.AddIssueComment(ctx, fullID, currentActor(), reason); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to add reason comment: %v\n", err) } } diff --git a/cmd/bd/unclaim_proxied_server.go b/cmd/bd/unclaim_proxied_server.go index 85b56acc2..9269beb88 100644 --- a/cmd/bd/unclaim_proxied_server.go +++ b/cmd/bd/unclaim_proxied_server.go @@ -62,9 +62,9 @@ func runUnclaimProxiedServer(ctx context.Context, args []string, reason string, var uerr error if expectedAssignee != "" { - uerr = uw.IssueUseCase().UnclaimIfAssignee(ctx, fullID, actor, expectedAssignee) + uerr = uw.IssueUseCase().UnclaimIfAssignee(ctx, fullID, currentActor(), expectedAssignee) } else { - uerr = uw.IssueUseCase().Unclaim(ctx, fullID, actor, force) + uerr = uw.IssueUseCase().Unclaim(ctx, fullID, currentActor(), force) } if uerr != nil { r.errs = append(r.errs, fmt.Sprintf("Error unclaiming %s: %v", fullID, uerr)) @@ -72,7 +72,7 @@ func runUnclaimProxiedServer(ctx context.Context, args []string, reason string, } if reason != "" { - if _, cerr := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, actor, reason); cerr != nil { + if _, cerr := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, currentActor(), reason); cerr != nil { r.errs = append(r.errs, fmt.Sprintf("Warning: failed to add reason comment on %s: %v", fullID, cerr)) } } @@ -129,7 +129,7 @@ func runReclaimProxiedServer(ctx context.Context, olderThan time.Duration, filte } reclaimed, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) ([]types.ReclaimedLease, string, error) { - out, rerr := uw.IssueUseCase().ReclaimExpiredLeases(ctx, olderThan, filter, actor) + out, rerr := uw.IssueUseCase().ReclaimExpiredLeases(ctx, olderThan, filter, currentActor()) if rerr != nil { return nil, "", rerr } diff --git a/cmd/bd/undefer.go b/cmd/bd/undefer.go index 08e251c6a..b89907be8 100644 --- a/cmd/bd/undefer.go +++ b/cmd/bd/undefer.go @@ -89,7 +89,7 @@ Examples: updates["status"] = string(types.StatusOpen) } - if err := store.UpdateIssue(ctx, fullID, updates, actor); err != nil { + if err := store.UpdateIssue(ctx, fullID, updates, currentActor()); err != nil { fmt.Fprintf(os.Stderr, "Error undeferring %s: %v\n", fullID, err) continue } diff --git a/cmd/bd/update.go b/cmd/bd/update.go index 1510b8c78..09e8edb64 100644 --- a/cmd/bd/update.go +++ b/cmd/bd/update.go @@ -511,7 +511,7 @@ pointless).`, // the actor's own fresh claim. A policy refusal, so it exits 1, // not 13. if newAssignee, ok := updates["assignee"].(string); ok && ifAssignee == nil && !claimFlag && !ifRevisionAlreadyStale(issue, ifRevision) { - if err := validateIssueReassignable(id, issue, actor, newAssignee, + if err := validateIssueReassignable(id, issue, currentActor(), newAssignee, storeClaimPoolAliases(ctx, issueStore), forceFlag); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) recordFailure(id, err.Error()) @@ -552,7 +552,7 @@ pointless).`, // through: `--force -s closed` is now a legitimate way to ask for // the close-policy half alone. updateResult, updateErr := runCommandUpdateMutation(opsCtx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: result.ResolvedID, patch: patch, claim: claimFlag, @@ -602,13 +602,13 @@ pointless).`, } // Audit log key field changes (survives Dolt GC flatten) if patch.Status.Set { - audit.LogFieldChange(result.ResolvedID, "status", string(issue.Status), string(patch.Status.Value), actor, "") + audit.LogFieldChange(result.ResolvedID, "status", string(issue.Status), string(patch.Status.Value), currentActor(), "") } if patch.Assignee.Set { - audit.LogFieldChange(result.ResolvedID, "assignee", issue.Assignee, patch.Assignee.Value, actor, "") + audit.LogFieldChange(result.ResolvedID, "assignee", issue.Assignee, patch.Assignee.Value, currentActor(), "") } if patch.Priority.Set { - audit.LogFieldChange(result.ResolvedID, "priority", fmt.Sprintf("%d", issue.Priority), fmt.Sprintf("%d", patch.Priority.Value), actor, "") + audit.LogFieldChange(result.ResolvedID, "priority", fmt.Sprintf("%d", issue.Priority), fmt.Sprintf("%d", patch.Priority.Value), currentActor(), "") } // The operation's own post-state snapshot replaces the re-read. @@ -642,7 +642,7 @@ pointless).`, if s == nil { continue } - if err := commitPendingIfEmbedded(ctx, s, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, s, currentActor(), doltAutoCommitParams{ Command: "update", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/update_proxied_server.go b/cmd/bd/update_proxied_server.go index fee3e9225..5d74583b2 100644 --- a/cmd/bd/update_proxied_server.go +++ b/cmd/bd/update_proxied_server.go @@ -127,7 +127,7 @@ func applyUpdateProxiedOne(ctx context.Context, id string, in *updateInput) (*ty expectedStatus = &expected } result, err := runCommandUpdateMutation(ctx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: id, patch: patch, claim: in.claim, @@ -226,7 +226,7 @@ func proxiedUpdateTarget(ctx context.Context, id string, in *updateInput) (*type // mc-zndi7.74: also skipped when this pre-read is already stale against an // active --if-revision guard — see ifRevisionAlreadyStale's doc. if newAssignee, ok := in.fields["assignee"].(string); ok && in.ifAssignee == nil && !in.claim && !ifRevisionAlreadyStale(current, in.ifRevision) { - if err := validateIssueReassignable(id, current, actor, newAssignee, + if err := validateIssueReassignable(id, current, currentActor(), newAssignee, proxiedClaimPoolAliases(ctx), in.force); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) return nil, &updateIDFailure{ID: id, Error: err.Error()} diff --git a/cmd/bd/wisp.go b/cmd/bd/wisp.go index 329b1734f..a1b7ac27e 100644 --- a/cmd/bd/wisp.go +++ b/cmd/bd/wisp.go @@ -280,7 +280,7 @@ func runWispCreateCore(cmd *cobra.Command, args []string) error { result, err := spawnMoleculeWithOptions(ctx, store, subgraph, CloneOptions{ Vars: vars, - Actor: actor, + Actor: currentActor(), Ephemeral: true, Prefix: types.IDPrefixWisp, RootOnly: rootOnly, diff --git a/cmd/bd/wisp_proxied_server.go b/cmd/bd/wisp_proxied_server.go index edc727ea8..fd43df23b 100644 --- a/cmd/bd/wisp_proxied_server.go +++ b/cmd/bd/wisp_proxied_server.go @@ -92,7 +92,7 @@ func runWispCreateProxiedServer(ctx context.Context, in wispCreateInput) error { w := newUOWMolWriter(uw) spawnResult, err := cloneSubgraphInto(ctx, w, subgraph, CloneOptions{ Vars: vars, - Actor: actor, + Actor: currentActor(), Ephemeral: true, Prefix: types.IDPrefixWisp, RootOnly: in.rootOnly, @@ -189,7 +189,7 @@ func runWispGCProxiedServer(ctx context.Context, dryRun bool, ageThreshold time. IDs: ids, Cascade: false, UpdateTextReferences: true, - }, actor) + }, currentActor()) if err != nil { return domain.DeleteIssuesResult{}, "", err } @@ -320,7 +320,7 @@ func runWispPurgeClosedProxiedServer(ctx context.Context, dryRun, force bool, ex IDs: ids, Cascade: false, UpdateTextReferences: true, - }, actor) + }, currentActor()) if err != nil { return domain.DeleteIssuesResult{}, "", err } diff --git a/internal/beads/context.go b/internal/beads/context.go index 63c63394c..7db3b0ee8 100644 --- a/internal/beads/context.go +++ b/internal/beads/context.go @@ -314,13 +314,23 @@ func isExternalBeadsDir(beadsDir string) (bool, error) { // getGitCommonDirForPath returns the shared git directory for a path. // For worktrees, this returns the shared git directory (common to all worktrees). +// +// Every startup that builds a RepoContext asks this about the beads +// directory, so the answer comes from git.CommonDirInProcess when it can, +// with `git -C path rev-parse --git-common-dir` as the fallback. func getGitCommonDirForPath(path string) (string, error) { - cmd := exec.Command("git", "-C", path, "rev-parse", "--git-common-dir") - output, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("failed to get git common dir for %s: %w", path, err) + result, isRepo, ok := git.CommonDirInProcess(path) + switch { + case ok && !isRepo: + return "", fmt.Errorf("failed to get git common dir for %s: not a git repository", path) + case !ok: + cmd := exec.Command("git", "-C", path, "rev-parse", "--git-common-dir") + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to get git common dir for %s: %w", path, err) + } + result = strings.TrimSpace(string(output)) } - result := strings.TrimSpace(string(output)) if !filepath.IsAbs(result) { absPath, err := filepath.Abs(path) diff --git a/internal/git/BUILD.bazel b/internal/git/BUILD.bazel index 4d259cd16..e36e0ec55 100644 --- a/internal/git/BUILD.bazel +++ b/internal/git/BUILD.bazel @@ -2,7 +2,12 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "git", - srcs = ["gitdir.go"], + srcs = [ + "discover_linux.go", + "discover_other.go", + "gitconfig_linux.go", + "gitdir.go", + ], importpath = "github.com/steveyegge/beads/internal/git", visibility = ["//:__subpackages__"], ) @@ -10,6 +15,7 @@ go_library( go_test( name = "git_test", srcs = [ + "discover_linux_test.go", "gitdir_test.go", "jujutsu_test.go", ], diff --git a/internal/git/discover_linux.go b/internal/git/discover_linux.go new file mode 100644 index 000000000..4f88fd903 --- /dev/null +++ b/internal/git/discover_linux.go @@ -0,0 +1,559 @@ +//go:build linux + +package git + +import ( + "bytes" + "errors" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "syscall" +) + +// discoveryEnvOverrides are the environment variables that change how Git +// finds or validates a repository. When any is present, discovery is left to +// git itself. The list is deliberately broader than strictly necessary: a +// fallback costs one subprocess, a wrong answer costs a wrong workspace. +var discoveryEnvOverrides = []string{ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_CEILING_DIRECTORIES", + "GIT_DISCOVERY_ACROSS_FILESYSTEM", + "GIT_OBJECT_DIRECTORY", + "GIT_CONFIG_PARAMETERS", + "GIT_CONFIG_COUNT", + "GIT_TEST_ASSUME_DIFFERENT_OWNER", +} + +// discoverGitInProcess answers `git rev-parse --git-dir --git-common-dir +// --show-toplevel` for the process working directory without running git. ok +// is false whenever the layout is outside what it reproduces exactly; the +// caller then runs git. +func discoverGitInProcess() (revParseResult, bool) { + if discoveryEnvOverridden() { + return revParseResult{}, false + } + wd, err := processCwd() + if err != nil { + return revParseResult{}, false + } + return discoverGitFrom(wd) +} + +// processCwd is getcwd(2), as Git uses. os.Getwd prefers $PWD whenever it +// names the same inode as ".", which under bind-mount aliases (same device +// and inode, different ancestors) would walk a different path than git. +func processCwd() (string, error) { + return syscall.Getwd() +} + +// errNotGitRepositoryMountBoundary is the in-process counterpart of Git's +// "not a git repository (or any parent up to mount point ...)". +var errNotGitRepositoryMountBoundary = errors.New("no .git found in the working directory or any parent up to the filesystem boundary") + +// discoverGitFrom mirrors setup_git_directory_gently's discovery walk (Git +// 2.x) from cwd. Git walks the physical directory (getcwd), checks each +// directory's .git as a gitfile or a git directory, then the directory itself +// as a bare repository, and stops at the first filesystem boundary. +func discoverGitFrom(cwd string) (revParseResult, bool) { + cwd, err := filepath.EvalSymlinks(cwd) + if err != nil || !filepath.IsAbs(cwd) { + return revParseResult{}, false + } + cwd = filepath.Clean(cwd) + cwdStat := statOf(cwd) + if cwdStat == nil { + return revParseResult{}, false + } + + for dir := cwd; ; { + dotGit := filepath.Join(dir, ".git") + info, err := os.Lstat(dotGit) + switch { + case err == nil && info.Mode().IsDir(): + return discoveredGitDir(cwd, dir, dotGit) + case err == nil && info.Mode().IsRegular(): + return discoveredGitFile(dir, dotGit) + case err == nil: + // Symlinked .git, sockets, ...: let git decide. + return revParseResult{}, false + case !os.IsNotExist(err): + return revParseResult{}, false + } + // The directory itself may be a git directory (a bare repository, or + // somewhere inside a .git directory); git answers those differently. + if looksLikeGitDir(dir) { + return revParseResult{}, false + } + + parent := filepath.Dir(dir) + if parent == dir { + return revParseResult{notRepo: true}, true + } + parentStat := statOf(parent) + if parentStat == nil { + return revParseResult{}, false + } + if parentStat.Dev != cwdStat.Dev { + // Git stops at a mount point unless + // GIT_DISCOVERY_ACROSS_FILESYSTEM is set (handled by the caller). + return revParseResult{notRepo: true, notRepoErr: errNotGitRepositoryMountBoundary}, true + } + dir = parent + } +} + +// discoveredGitDir handles /.git being a directory. +func discoveredGitDir(cwd, top, gitDir string) (revParseResult, bool) { + // A .git directory carrying a commondir file is unusual; leave it to git. + if exists(filepath.Join(gitDir, "commondir")) { + return revParseResult{}, false + } + if !isGitDirectory(gitDir, gitDir) { + // Git would keep walking upward past an invalid .git directory. + return revParseResult{}, false + } + if !ownedByCurrentUser(top) || !ownedByCurrentUser(gitDir) { + return revParseResult{}, false + } + if worktree, ok := repoConfigAllowsDiscovery(gitDir); !ok || worktree != "" { + return revParseResult{}, false + } + if cwd == top { + return revParseResult{gitDir: ".git", commonDir: ".git", topLevel: top}, true + } + // From a subdirectory Git prints the git directory absolute and the common + // directory relative to the working directory. + rel, err := filepath.Rel(top, cwd) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + return revParseResult{}, false + } + depth := len(strings.Split(rel, string(filepath.Separator))) + return revParseResult{ + gitDir: gitDir, + commonDir: strings.Repeat("../", depth) + ".git", + topLevel: top, + }, true +} + +// discoveredGitFile handles /.git being a gitfile ("gitdir: "), as +// in linked worktrees and submodules. +func discoveredGitFile(top, gitFile string) (revParseResult, bool) { + data, err := os.ReadFile(gitFile) // #nosec G304 -- the .git file Git itself would read during discovery + if err != nil { + return revParseResult{}, false + } + content := strings.TrimRight(string(data), "\r\n") + target, found := strings.CutPrefix(content, "gitdir: ") + if !found || target == "" || strings.ContainsAny(target, "\n\r\x00") { + return revParseResult{}, false + } + if !filepath.IsAbs(target) { + // Not filepath.Join: its lexical Clean would fold "link/.." before + // symlinks resolve, where Git's realpath resolves them physically + // (as EvalSymlinks does for ".." after a resolved component). + target = top + string(filepath.Separator) + target + } + gitDir, err := filepath.EvalSymlinks(target) + if err != nil { + return revParseResult{}, false + } + commonDir := gitDir + if raw, err := os.ReadFile(filepath.Join(gitDir, "commondir")); err == nil { + common := strings.TrimRight(string(raw), "\r\n") + if common == "" || strings.ContainsAny(common, "\n\r\x00") { + return revParseResult{}, false + } + if !filepath.IsAbs(common) { + common = gitDir + string(filepath.Separator) + common // physical "..", as above + } + if commonDir, err = filepath.EvalSymlinks(common); err != nil { + return revParseResult{}, false + } + } else if !os.IsNotExist(err) { + return revParseResult{}, false + } + if !isGitDirectory(gitDir, commonDir) { + return revParseResult{}, false + } + if !ownedByCurrentUser(gitFile) || !ownedByCurrentUser(top) || !ownedByCurrentUser(gitDir) { + return revParseResult{}, false + } + worktree, ok := repoConfigAllowsDiscovery(commonDir) + if !ok { + return revParseResult{}, false + } + if worktree != "" && !submoduleWorktreeIs(gitDir, commonDir, worktree, top) { + return revParseResult{}, false + } + // Git prints both absolute from anywhere in the work tree. + return revParseResult{gitDir: gitDir, commonDir: commonDir, topLevel: top}, true +} + +// isGitDirectory mirrors Git's is_git_directory: a valid HEAD in the git +// directory, objects/ and refs/ in the common directory. +func isGitDirectory(gitDir, commonDir string) bool { + if !validHeadRef(filepath.Join(gitDir, "HEAD")) { + return false + } + return isDir(filepath.Join(commonDir, "objects")) && isDir(filepath.Join(commonDir, "refs")) +} + +// looksLikeGitDir is a cheap superset test for "git might treat dir as a git +// directory"; a hit only means discovery is handed to git. +func looksLikeGitDir(dir string) bool { + return exists(filepath.Join(dir, "HEAD")) && + (exists(filepath.Join(dir, "objects")) || exists(filepath.Join(dir, "commondir"))) +} + +// validHeadRef accepts the HEAD forms Git's validate_headref does for the +// files it writes itself: "ref: refs/..." or a full object id. Symlinked HEAD +// (an ancient layout) is left to git. +func validHeadRef(path string) bool { + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() { + return false + } + data, err := os.ReadFile(path) // #nosec G304 -- HEAD of the discovered git directory + if err != nil { + return false + } + if ref, ok := bytes.CutPrefix(data, []byte("ref:")); ok { + return bytes.HasPrefix(bytes.TrimLeft(ref, " \t"), []byte("refs/")) + } + line := strings.TrimRight(string(data), "\r\n") + if len(line) != 40 && len(line) != 64 { + return false + } + for _, c := range line { + if !strings.ContainsRune("0123456789abcdef", c) { + return false + } + } + return true +} + +// submoduleWorktreeIs reports whether a gitfile repository's core.worktree +// names exactly the directory holding the gitfile. Git writes that for every +// submodule (core.worktree = ../../../sub in .git/modules/sub/config), and Git +// then resolves the work tree by chdir(gitdir) + chdir(core.worktree), which +// lands on top again. Only that plain shape is accepted: a separate git +// directory (no shared common directory) and a value of leading "../" +// segments followed by plain names, which resolves lexically from the +// already-physical git directory exactly as the chdirs do. +func submoduleWorktreeIs(gitDir, commonDir, worktree, top string) bool { + if gitDir != commonDir || filepath.IsAbs(worktree) { + return false + } + rest := filepath.ToSlash(filepath.Clean(worktree)) + if rest != worktree && rest+"/" != worktree { + return false + } + for strings.HasPrefix(rest, "../") { + rest = rest[len("../"):] + } + for _, part := range strings.Split(rest, "/") { + if part == "" || part == "." || part == ".." { + return false + } + } + resolved, err := filepath.EvalSymlinks(filepath.Join(gitDir, worktree)) + return err == nil && resolved == top +} + +// repoConfigAllowsDiscovery reports whether the repository config leaves +// Git's answer to the plain discovery result, and returns core.worktree when +// it is set (the caller decides whether it is the submodule shape). Git +// consults only the repository's own config file here +// (read_repository_format), so global and system config cannot move the work +// tree. Anything else that could — core.bare, extensions (worktreeConfig, +// unknown ones Git rejects), a format version Git refuses, include +// directives, a repeated core.worktree, or any line scanGitConfig does not +// fully understand (where Git might honor or reject it) — sends discovery to +// git. +func repoConfigAllowsDiscovery(commonDir string) (worktree string, ok bool) { + ok = scanGitConfig(filepath.Join(commonDir, "config"), func(e gitConfigEntry) bool { + if strings.HasPrefix(e.section, "include") || e.section == "extensions" { + return false + } + if e.section != "core" || e.subsection != nil { + return true + } + switch e.key { + case "worktree": + if worktree != "" || !e.hasValue || e.value == "" || strings.ContainsAny(e.rawValue, "\"#;") { + return false + } + worktree = e.value + case "bare": + if !e.hasValue { + return false // a bare "bare" key means true + } + switch strings.ToLower(e.value) { + case "false", "no", "off", "0": + default: + return false + } + case "repositoryformatversion": + if v, err := strconv.Atoi(e.value); err != nil || v < 0 || v > 1 { + return false + } + } + return true + }) + if !ok { + return "", false + } + return worktree, true +} + +// ownedByCurrentUser mirrors Git's is_path_owned_by_current_uid (lstat, owner +// equals the effective uid). Git's sudo allowance (root with SUDO_UID) is left +// to git. A path owned by someone else engages safe.directory, which only git +// evaluates. +func ownedByCurrentUser(path string) bool { + euid := os.Geteuid() + if euid == 0 { + if _, set := os.LookupEnv("SUDO_UID"); set { + return false + } + } + info, err := os.Lstat(path) + if err != nil { + return false + } + st, ok := info.Sys().(*syscall.Stat_t) + return ok && int(st.Uid) == euid +} + +// statOf returns path's stat data (following symlinks, as Git's +// get_device_or_die does), or nil. +func statOf(path string) *syscall.Stat_t { + info, err := os.Stat(path) + if err != nil { + return nil + } + st, _ := info.Sys().(*syscall.Stat_t) + return st +} + +func exists(path string) bool { + _, err := os.Lstat(path) + return err == nil +} + +func isDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + +// CommonDirInProcess answers `git -C dir rev-parse --git-common-dir` without +// running git, in git's own spelling (relative to dir's physical location +// when git prints it relative). isRepo is false where git would fail with +// "not a git repository". ok is false whenever discoverGitInProcess would +// decline; the caller then runs git. +func CommonDirInProcess(dir string) (commonDir string, isRepo, ok bool) { + if discoveryEnvOverridden() { + return "", false, false + } + abs, err := filepath.Abs(dir) + if err != nil { + return "", false, false + } + raw, ok := discoverGitFrom(abs) + if !ok { + return "", false, false + } + if raw.notRepo { + return "", false, true + } + return raw.commonDir, true, true +} + +// HasRemoteInProcess answers "does `git remote` in dir print anything" (dir +// "" is the process working directory) without running git, or ok=false. +// +// `git remote` lists every remote that any config scope defines a +// remote..* key for, and fails outside a repository. The answer is +// given only when every file git would read — system, global (XDG and +// ~/.gitconfig, or GIT_CONFIG_GLOBAL), and the repository's own config — is +// absent or within scanGitConfig's grammar and free of include directives, +// the repository is one discoverGitFrom answers for, and no environment +// variable adds config or redirects discovery. The system file's location is +// compiled into git; the candidates are /etc/gitconfig and /etc/gitconfig +// for the git found on PATH (as invoked and with symlinks resolved), and a +// remote seen only in a system candidate declines rather than guess which one +// git reads. +func HasRemoteInProcess(dir string) (has, ok bool) { + if discoveryEnvOverridden() { + return false, false + } + if _, set := os.LookupEnv("GIT_CONFIG"); set { + return false, false + } + if dir == "" { + wd, err := processCwd() + if err != nil { + return false, false + } + dir = wd + } + abs, err := filepath.Abs(dir) + if err != nil { + return false, false + } + physical, err := filepath.EvalSymlinks(abs) + if err != nil { + return false, false + } + raw, ok := discoverGitFrom(physical) + if !ok { + return false, false + } + if raw.notRepo { + return false, true // `git remote` exits 128 outside a repository + } + commonDir := raw.commonDir + if !filepath.IsAbs(commonDir) { + commonDir = filepath.Join(physical, commonDir) + } + // Legacy remote definitions (.git/remotes/*, .git/branches/*) are not + // listed by `git remote`, but are not modeled either: decline if present. + for _, legacy := range []string{"remotes", "branches"} { + if entries, err := os.ReadDir(filepath.Join(commonDir, legacy)); err == nil && len(entries) > 0 { + return false, false + } else if err != nil && !os.IsNotExist(err) { + return false, false + } + } + + system, ok := systemConfigCandidates() + if !ok { + return false, false + } + global, ok := globalConfigFiles() + if !ok { + return false, false + } + inSystem, inOther := false, false + for _, scope := range []struct { + files []string + system bool + }{{system, true}, {global, false}, {[]string{filepath.Join(commonDir, "config")}, false}} { + for _, file := range scope.files { + found, ok := configDefinesRemote(file) + if !ok { + return false, false + } + if found && scope.system { + inSystem = true + } else if found { + inOther = true + } + } + } + switch { + case inOther: + return true, true + case inSystem: + return false, false + default: + return false, true + } +} + +// remoteName is the remote-name shape answered in-process; git ignores names +// starting with "/" and accepts many others this does not model. +var remoteName = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) + +// configDefinesRemote reports whether file defines a remote..* key. +func configDefinesRemote(file string) (found, ok bool) { + ok = scanGitConfig(file, func(e gitConfigEntry) bool { + if strings.HasPrefix(e.section, "include") { + return false + } + if e.section != "remote" || e.subsection == nil { + return true // remote.pushDefault and friends define no remote + } + if !remoteName.MatchString(*e.subsection) { + return false + } + found = true + return true + }) + return found, ok +} + +// globalConfigFiles mirrors git_global_config. +func globalConfigFiles() ([]string, bool) { + if global, set := os.LookupEnv("GIT_CONFIG_GLOBAL"); set { + if global == "" || !filepath.IsAbs(global) { + return nil, false + } + return []string{global}, true + } + home := os.Getenv("HOME") + if home == "" || !filepath.IsAbs(home) { + return nil, false + } + xdg := filepath.Join(home, ".config", "git", "config") + if x := os.Getenv("XDG_CONFIG_HOME"); x != "" { + if !filepath.IsAbs(x) { + return nil, false + } + xdg = filepath.Join(x, "git", "config") + } + return []string{xdg, filepath.Join(home, ".gitconfig")}, true +} + +// systemConfigCandidates mirrors git_system_config, with the compiled-in +// ETC_GITCONFIG approximated as described on HasRemoteInProcess. +func systemConfigCandidates() ([]string, bool) { + if v, set := os.LookupEnv("GIT_CONFIG_NOSYSTEM"); set { + switch strings.ToLower(v) { + case "1", "true", "yes", "on": + return nil, true + case "", "0", "false", "no", "off": + default: + return nil, false + } + } + if system, set := os.LookupEnv("GIT_CONFIG_SYSTEM"); set { + if system == "" || !filepath.IsAbs(system) { + return nil, false + } + return []string{system}, true + } + bin, err := exec.LookPath("git") + if err != nil || !filepath.IsAbs(bin) { + return nil, false + } + candidates := []string{"/etc/gitconfig"} + paths := []string{bin} + if resolved, err := filepath.EvalSymlinks(bin); err == nil { + paths = append(paths, resolved) + } + for _, p := range paths { + if filepath.Base(filepath.Dir(p)) != "bin" { + return nil, false + } + if prefix := filepath.Dir(filepath.Dir(p)); prefix != "/usr" { + candidates = append(candidates, filepath.Join(prefix, "etc", "gitconfig")) + } + } + return candidates, true +} + +func discoveryEnvOverridden() bool { + for _, name := range discoveryEnvOverrides { + if _, set := os.LookupEnv(name); set { + return true + } + } + return false +} diff --git a/internal/git/discover_linux_test.go b/internal/git/discover_linux_test.go new file mode 100644 index 000000000..04d111f25 --- /dev/null +++ b/internal/git/discover_linux_test.go @@ -0,0 +1,471 @@ +//go:build linux + +package git + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// discoveryFixtureEnv is the environment fixture git runs under: the process +// environment minus every discovery override, with no system or user config. +func discoveryFixtureEnv(t *testing.T) []string { + t.Helper() + var env []string + for _, kv := range os.Environ() { + name, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(name, "GIT_") || name == "HOME" { + continue + } + env = append(env, kv) + } + return append(env, "HOME="+t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com") +} + +func runFixtureGit(t *testing.T, env []string, dir string, args ...string) string { + t.Helper() + cmd := gitCommand(args...) + cmd.Dir, cmd.Env = dir, env + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s in %s: %v\n%s", strings.Join(args, " "), dir, err, out) + } + return string(out) +} + +// gitRevParse is the reference answer discoverGitFrom must reproduce. +func gitRevParse(t *testing.T, env []string, dir string) revParseResult { + t.Helper() + cmd := exec.Command("git", "rev-parse", "--git-dir", "--git-common-dir", "--show-toplevel") + cmd.Dir, cmd.Env = dir, env + out, err := cmd.Output() + if err != nil { + return revParseResult{notRepo: true} + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + if len(lines) != 3 { + t.Fatalf("git rev-parse in %s: unexpected output %q", dir, out) + } + return revParseResult{gitDir: lines[0], commonDir: lines[1], topLevel: lines[2]} +} + +type discoveryFixture struct { + root, main, worktree, relWorktree, symWorktree, submodule, nested string + env []string +} + +func newDiscoveryFixture(t *testing.T) discoveryFixture { + t.Helper() + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + env := discoveryFixtureEnv(t) + f := discoveryFixture{ + root: root, + main: filepath.Join(root, "main"), + worktree: filepath.Join(root, "wt"), + relWorktree: filepath.Join(root, "wt-rel"), + symWorktree: filepath.Join(root, "wt-sym"), + env: env, + } + f.submodule = filepath.Join(f.main, "sub") + f.nested = filepath.Join(f.main, "a", "inner") + subSrc := filepath.Join(root, "subsrc") + for _, dir := range []string{f.main, subSrc, filepath.Join(f.main, "a", "b"), f.nested, filepath.Join(root, "norepo", "x")} { + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + for _, dir := range []string{f.main, subSrc} { + runFixtureGit(t, env, dir, "init", "-q") + runFixtureGit(t, env, dir, "commit", "-q", "--allow-empty", "-m", "base") + } + runFixtureGit(t, env, f.main, "worktree", "add", "-q", f.worktree) + runFixtureGit(t, env, f.main, "worktree", "add", "-q", f.relWorktree) + // A relative gitfile, as `git worktree add --relative-paths` (or a moved + // checkout) writes it. + if err := os.WriteFile(filepath.Join(f.relWorktree, ".git"), []byte("gitdir: ../main/.git/worktrees/wt-rel\n"), 0o600); err != nil { + t.Fatal(err) + } + // A gitfile naming its git directory through a symlink: Git prints the + // resolved path. + runFixtureGit(t, env, f.main, "worktree", "add", "-q", f.symWorktree) + if err := os.Symlink(filepath.Join(f.main, ".git"), filepath.Join(root, "gitlink")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(f.symWorktree, ".git"), []byte("gitdir: "+filepath.Join(root, "gitlink", "worktrees", "wt-sym")+"\n"), 0o600); err != nil { + t.Fatal(err) + } + runFixtureGit(t, env, f.main, "-c", "protocol.file.allow=always", "submodule", "add", "-q", subSrc, "sub") + runFixtureGit(t, env, f.nested, "init", "-q") + for _, dir := range []string{filepath.Join(f.worktree, "w"), filepath.Join(f.submodule, "s"), filepath.Join(f.nested, "deep")} { + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(f.main, filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + return f +} + +// resolvedLikeStartup is what bd's startup derives for dir: the in-process +// answer when discovery gives one, else git's. +func resolvedLikeStartup(dir string, env []string) (gitContext, bool) { + if raw, ok := discoverGitFrom(dir); ok { + return gitContextFromRevParse(dir, raw), true + } + return loadGitContext(dir, env), false +} + +// assertSameAsGit checks the startup-derived context for dir against the git +// subprocess's: same failure, or the same paths. +func assertSameAsGit(t *testing.T, env []string, dir string) { + t.Helper() + got, _ := resolvedLikeStartup(dir, env) + want := loadGitContext(dir, env) + if (got.err != nil) != (want.err != nil) { + t.Fatalf("in %s: startup err %v, git err %v", dir, got.err, want.err) + } + if got.gitDirRaw != want.gitDirRaw || got.commonDir != want.commonDir || + got.repoRoot != want.repoRoot || got.isWorktree != want.isWorktree { + t.Fatalf("in %s: startup %+v, git %+v", dir, got, want) + } +} + +// repoWithRawConfig makes a repository and replaces its config file. +func repoWithRawConfig(t *testing.T, env []string, config string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "r") + runFixtureGit(t, env, t.TempDir(), "init", "-q", dir) + if err := os.WriteFile(filepath.Join(dir, ".git", "config"), []byte(config), 0o600); err != nil { + t.Fatal(err) + } + return dir +} + +const plainRepoConfig = "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n\tlogallrefupdates = true\n" + +// TestDiscoverGitFromMatchesRevParse is the equivalence contract: wherever +// in-process discovery answers, its answer is byte-for-byte what `git +// rev-parse --git-dir --git-common-dir --show-toplevel` prints there. +func TestDiscoverGitFromMatchesRevParse(t *testing.T) { + f := newDiscoveryFixture(t) + for _, tc := range []struct{ name, dir string }{ + {"repo top", f.main}, + {"repo subdir", filepath.Join(f.main, "a", "b")}, + {"repo first-level subdir", filepath.Join(f.main, "a")}, + {"linked worktree top", f.worktree}, + {"linked worktree subdir", filepath.Join(f.worktree, "w")}, + {"linked worktree with relative gitfile", f.relWorktree}, + {"linked worktree with gitfile through a symlink", f.symWorktree}, + {"submodule top", f.submodule}, + {"submodule subdir", filepath.Join(f.submodule, "s")}, + {"nested repo top", f.nested}, + {"nested repo subdir", filepath.Join(f.nested, "deep")}, + {"symlinked path to repo top", filepath.Join(f.root, "link")}, + {"symlinked path to repo subdir", filepath.Join(f.root, "link", "a", "b")}, + // No ancestor of a fresh temp directory is a repository, so this is + // answered in-process (by the walk reaching / or a mount boundary). + {"no repository", filepath.Join(f.root, "norepo", "x")}, + {"config with a UTF-8 BOM", repoWithRawConfig(t, f.env, "\xef\xbb\xbf"+plainRepoConfig)}, + {"config with CRLF and comments", repoWithRawConfig(t, f.env, + "# leading comment\r\n[core] ; header comment\r\n\tbare = false # trailing\r\n[remote \"origin\"]\r\n\turl = \"/x y\"\r\n")}, + } { + t.Run(tc.name, func(t *testing.T) { + got, ok := discoverGitFrom(tc.dir) + if !ok { + t.Fatalf("discoverGitFrom(%s) declined; want an in-process answer", tc.dir) + } + if want := gitRevParse(t, f.env, tc.dir); got != want { + t.Fatalf("discoverGitFrom(%s) = %+v, git rev-parse = %+v", tc.dir, got, want) + } + assertSameAsGit(t, f.env, tc.dir) + }) + } +} + +// TestDiscoverGitFromDeclinesUnmodeledLayouts covers the layouts in-process +// discovery must hand to git rather than guess, and checks that what startup +// then derives (from git) is git's answer. +func TestDiscoverGitFromDeclinesUnmodeledLayouts(t *testing.T) { + f := newDiscoveryFixture(t) + repoWithConfig := func(t *testing.T, args ...string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "r") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", dir) + runFixtureGit(t, f.env, dir, append([]string{"config"}, args...)...) + return dir + } + mkdir := func(t *testing.T, parts ...string) string { + t.Helper() + dir := filepath.Join(parts...) + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + return dir + } + invalidGitfile := mkdir(t, t.TempDir(), "badfile") + if err := os.WriteFile(filepath.Join(invalidGitfile, ".git"), []byte("not a gitfile\n"), 0o600); err != nil { + t.Fatal(err) + } + symlinkedDotGit := mkdir(t, t.TempDir(), "symlinked") + if err := os.Symlink(filepath.Join(f.main, ".git"), filepath.Join(symlinkedDotGit, ".git")); err != nil { + t.Fatal(err) + } + bare := filepath.Join(t.TempDir(), "bare.git") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", "--bare", bare) + bareSub := mkdir(t, bare, "sub") // below a bare repository: the walk meets HEAD+objects + commondirInDotGit := repoWithConfig(t, "core.filemode", "true") + if err := os.WriteFile(filepath.Join(commondirInDotGit, ".git", "commondir"), []byte(".\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ name, dir string }{ + {"core.worktree", repoWithConfig(t, "core.worktree", t.TempDir())}, + {"core.bare", repoWithConfig(t, "core.bare", "true")}, + {"extensions.worktreeConfig", repoWithConfig(t, "extensions.worktreeConfig", "true")}, + {"include directive", repoWithConfig(t, "include.path", "/dev/null")}, + // Git skips a leading BOM and honors the section after it. + {"BOM then core.worktree", repoWithRawConfig(t, f.env, "\xef\xbb\xbf[core]\n\tworktree = "+t.TempDir()+"\n")}, + {"header with inner spaces", repoWithRawConfig(t, f.env, "[ core ]\n\tbare = false\n")}, + {"header with leading whitespace", repoWithRawConfig(t, f.env, " [core]\n\tbare = false\n")}, + {"malformed key line", repoWithRawConfig(t, f.env, plainRepoConfig+"!!! = x\n")}, + {"key before any section", repoWithRawConfig(t, f.env, "bare = false\n"+plainRepoConfig)}, + {"key on the header line", repoWithRawConfig(t, f.env, "[core] bare = false\n")}, + {"escaped value", repoWithRawConfig(t, f.env, plainRepoConfig+"[alias]\n\tx = \"a\\tb\"\n")}, + {"unterminated quote", repoWithRawConfig(t, f.env, plainRepoConfig+"[alias]\n\tx = \"a\n")}, + {"continuation line", repoWithRawConfig(t, f.env, plainRepoConfig+"[alias]\n\tx = a \\\n b\n")}, + {"dotted section", repoWithRawConfig(t, f.env, "[core.x]\n\ty = 1\n"+plainRepoConfig)}, + {"inside the git directory", filepath.Join(f.main, ".git", "refs")}, + {"bare repository", bare}, + {"below a bare repository", bareSub}, + {"commondir inside .git", commondirInDotGit}, + {"invalid gitfile", invalidGitfile}, + {"symlinked .git", symlinkedDotGit}, + } { + t.Run(tc.name, func(t *testing.T) { + if got, ok := discoverGitFrom(tc.dir); ok { + t.Fatalf("discoverGitFrom(%s) = %+v; want it to defer to git", tc.dir, got) + } + assertSameAsGit(t, f.env, tc.dir) + }) + } +} + +// TestDiscoverGitInProcessUsesPhysicalCwd: Git walks from getcwd(2); a $PWD +// naming the same directory through another path (a symlink here; a bind +// mount alias in containers) must not change the walk. +func TestDiscoverGitInProcessUsesPhysicalCwd(t *testing.T) { + f := newDiscoveryFixture(t) + physical := filepath.Join(f.main, "a", "b") + alias := filepath.Join(f.root, "link", "a", "b") + t.Chdir(physical) + t.Setenv("PWD", alias) + if wd, err := os.Getwd(); err != nil || wd != alias { + t.Fatalf("os.Getwd() = %q, %v; this test needs it to report the $PWD alias %q", wd, err, alias) + } + if wd, err := processCwd(); err != nil || wd != physical { + t.Fatalf("processCwd() = %q, %v; want getcwd's %q", wd, err, physical) + } + for _, k := range discoveryEnvOverrides { + if _, set := os.LookupEnv(k); set { + t.Setenv(k, "") + if err := os.Unsetenv(k); err != nil { + t.Fatal(err) + } + } + } + got, ok := discoverGitInProcess() + if !ok { + t.Fatal("discoverGitInProcess declined") + } + if want := gitRevParse(t, f.env, physical); got != want { + t.Fatalf("discoverGitInProcess() = %+v, git rev-parse = %+v", got, want) + } +} + +// TestDiscoverGitInProcessDefersToGitEnvOverrides: GIT_DIR and friends +// redefine discovery, so their presence (even empty) sends it to git. +func TestDiscoverGitInProcessDefersToGitEnvOverrides(t *testing.T) { + f := newDiscoveryFixture(t) + t.Chdir(f.main) + for _, name := range discoveryEnvOverrides { + t.Run(name, func(t *testing.T) { + t.Setenv(name, "") + if got, ok := discoverGitInProcess(); ok { + t.Fatalf("with %s set, discoverGitInProcess() = %+v; want it to defer to git", name, got) + } + }) + } +} + +// TestGitContextInProcessMatchesGitSubprocess checks the cached context end +// to end: what bd's startup now derives without git equals what it derived +// from the git subprocess, for each discovery layout. +func TestGitContextInProcessMatchesGitSubprocess(t *testing.T) { + f := newDiscoveryFixture(t) + for _, k := range discoveryEnvOverrides { + if _, set := os.LookupEnv(k); set { + t.Setenv(k, "") + if err := os.Unsetenv(k); err != nil { + t.Fatal(err) + } + } + } + for _, dir := range []string{ + f.main, filepath.Join(f.main, "a", "b"), f.worktree, filepath.Join(f.worktree, "w"), + f.submodule, filepath.Join(f.nested, "deep"), + } { + t.Run(strings.TrimPrefix(dir, f.root), func(t *testing.T) { + t.Chdir(dir) + raw, ok := discoverGitInProcess() + if !ok { + t.Fatalf("discoverGitInProcess declined in %s", dir) + } + got := gitContextFromRevParse("", raw) + want := loadGitContext("", f.env) + if got.err != nil || want.err != nil { + t.Fatalf("errors: in-process %v, git %v", got.err, want.err) + } + if got.gitDirRaw != want.gitDirRaw || got.commonDir != want.commonDir || + got.repoRoot != want.repoRoot || got.isWorktree != want.isWorktree { + t.Fatalf("in-process %+v, git %+v", got, want) + } + }) + } +} + +// isolateGitConfigEnv points git and the in-process readers at the same, +// controlled config files: no system config, an empty global file. +func isolateGitConfigEnv(t *testing.T) (global string) { + t.Helper() + for _, kv := range os.Environ() { + name, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(name, "GIT_") { + t.Setenv(name, "") + if err := os.Unsetenv(name); err != nil { + t.Fatal(err) + } + } + } + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "xdg")) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + global = filepath.Join(home, "global.gitconfig") + if err := os.WriteFile(global, nil, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("GIT_CONFIG_GLOBAL", global) + return global +} + +func gitRemoteListsAny(t *testing.T, dir string) bool { + t.Helper() + cmd := exec.Command("git", "remote") + cmd.Dir = dir + out, err := cmd.Output() + return err == nil && strings.TrimSpace(string(out)) != "" +} + +// TestHasRemoteInProcessMatchesGitRemote: wherever HasRemoteInProcess +// answers, it is what `git remote` (the auto-backup probe) reports. +func TestHasRemoteInProcessMatchesGitRemote(t *testing.T) { + f := newDiscoveryFixture(t) + global := isolateGitConfigEnv(t) + withRemote := filepath.Join(t.TempDir(), "with-remote") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", withRemote) + runFixtureGit(t, f.env, withRemote, "remote", "add", "origin", "https://example.invalid/r.git") + pushDefaultOnly := filepath.Join(t.TempDir(), "push-default") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", pushDefaultOnly) + runFixtureGit(t, f.env, pushDefaultOnly, "config", "remote.pushDefault", "origin") + runFixtureGit(t, f.env, f.main, "remote", "add", "up", "https://example.invalid/m.git") + + setGlobal := func(t *testing.T, content string) { + t.Helper() + if err := os.WriteFile(global, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.WriteFile(global, nil, 0o600) }) + } + for _, tc := range []struct { + name, dir, global string + wantDecline bool + }{ + {name: "no remote", dir: f.nested}, + {name: "local remote", dir: withRemote}, + {name: "remote.pushDefault only", dir: pushDefaultOnly}, + {name: "linked worktree sees the common config", dir: filepath.Join(f.worktree, "w")}, + {name: "submodule (origin in its own config)"}, + {name: "no repository", dir: filepath.Join(f.root, "norepo", "x")}, + {name: "remote in global config", dir: f.nested, global: "[remote \"g\"]\n\turl = /g\n"}, + {name: "include in global config", dir: f.nested, global: "[include]\n\tpath = /dev/null\n", wantDecline: true}, + {name: "malformed global config", dir: f.nested, global: "[ user ]\n", wantDecline: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := tc.dir + if dir == "" { + dir = f.submodule + } + if tc.global != "" { + setGlobal(t, tc.global) + } + has, ok := HasRemoteInProcess(dir) + if ok == tc.wantDecline { + t.Fatalf("HasRemoteInProcess(%s) ok=%v, want ok=%v", dir, ok, !tc.wantDecline) + } + if want := gitRemoteListsAny(t, dir); ok && has != want { + t.Fatalf("HasRemoteInProcess(%s) = %v, git remote lists any = %v", dir, has, want) + } + }) + } + t.Run("GIT_CONFIG_PARAMETERS declines", func(t *testing.T) { + t.Setenv("GIT_CONFIG_PARAMETERS", "'remote.x.url'='/x'") + if _, ok := HasRemoteInProcess(f.nested); ok { + t.Fatal("HasRemoteInProcess answered despite command-line config") + } + }) + t.Run("process cwd", func(t *testing.T) { + t.Chdir(withRemote) + if has, ok := HasRemoteInProcess(""); !ok || !has { + t.Fatalf("HasRemoteInProcess(\"\") = %v, %v; want true, true", has, ok) + } + }) +} + +// TestCommonDirInProcessMatchesGit: the auto-backup RepoContext asks +// `git -C <.beads> rev-parse --git-common-dir`; the in-process answer has +// git's spelling. +func TestCommonDirInProcessMatchesGit(t *testing.T) { + f := newDiscoveryFixture(t) + isolateGitConfigEnv(t) + beadsDir := filepath.Join(f.main, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + for _, dir := range []string{ + f.main, beadsDir, filepath.Join(f.main, "a", "b"), f.worktree, f.submodule, + filepath.Join(f.root, "link", "a"), filepath.Join(f.root, "norepo", "x"), + } { + t.Run(strings.TrimPrefix(dir, f.root), func(t *testing.T) { + got, isRepo, ok := CommonDirInProcess(dir) + if !ok { + t.Fatalf("CommonDirInProcess(%s) declined", dir) + } + cmd := exec.Command("git", "-C", dir, "rev-parse", "--git-common-dir") + out, err := cmd.Output() + if (err == nil) != isRepo { + t.Fatalf("CommonDirInProcess(%s) isRepo=%v, git err=%v", dir, isRepo, err) + } + if want := strings.TrimSpace(string(out)); isRepo && got != want { + t.Fatalf("CommonDirInProcess(%s) = %q, git = %q", dir, got, want) + } + }) + } +} diff --git a/internal/git/discover_other.go b/internal/git/discover_other.go new file mode 100644 index 000000000..231616c9b --- /dev/null +++ b/internal/git/discover_other.go @@ -0,0 +1,20 @@ +//go:build !linux + +package git + +// discoverGitInProcess is linux-only: elsewhere path case-folding (darwin, +// windows) and ownership semantics (windows) make an exact in-process answer +// harder to prove, so discovery always runs git. +func discoverGitInProcess() (revParseResult, bool) { + return revParseResult{}, false +} + +// CommonDirInProcess is linux-only; see discoverGitInProcess. +func CommonDirInProcess(string) (commonDir string, isRepo, ok bool) { + return "", false, false +} + +// HasRemoteInProcess is linux-only; see discoverGitInProcess. +func HasRemoteInProcess(string) (has, ok bool) { + return false, false +} diff --git a/internal/git/gitconfig_linux.go b/internal/git/gitconfig_linux.go new file mode 100644 index 000000000..ea786b373 --- /dev/null +++ b/internal/git/gitconfig_linux.go @@ -0,0 +1,109 @@ +//go:build linux + +package git + +import ( + "bytes" + "os" + "regexp" + "strings" +) + +// gitConfigEntry is one key line of a git config file, as scanGitConfig +// understood it. +type gitConfigEntry struct { + section string // lower-cased section name + subsection *string // nil for [section]; case preserved + key string // lower-cased variable name + hasValue bool // false for a bare "key" line (boolean true) + rawValue string // value text after '=', as written, comment included + value string // rawValue without its comment, trimmed and unquoted +} + +var ( + gitConfigHeader = regexp.MustCompile(`^\[([A-Za-z0-9-]+)(?: "([^"\\]*)")?\][ \t]*(?:[#;].*)?$`) + gitConfigKeyLine = regexp.MustCompile(`^[ \t]*([A-Za-z][A-Za-z0-9-]*)[ \t]*(?:=(.*)|(?:[#;].*)?)$`) + gitConfigCommentLn = regexp.MustCompile(`^[ \t]*(?:[#;].*)?$`) +) + +// scanGitConfig reads a git config file with a deliberately narrow grammar +// and calls visit for each key line; it returns false (the caller then asks +// git) when the file is unreadable, a visit returns false, or ANY line falls +// outside that grammar — Git might honor such a line differently or refuse +// the file. A missing file is an empty config, as for Git. +// +// Understood: a UTF-8 BOM at the start of the file (Git skips it), CRLF line +// endings, blank and comment lines, "[section]" and `[section "sub"]` +// headers at the start of a line (optionally followed by a comment), and +// "key", "key = value" lines inside a section. Values containing a backslash +// or an odd number of double quotes (escapes, continuations, unterminated +// strings) are not understood. +func scanGitConfig(path string, visit func(gitConfigEntry) bool) bool { + data, err := os.ReadFile(path) // #nosec G304 -- a git config file Git itself would read + if os.IsNotExist(err) { + return true + } + if err != nil || bytes.IndexByte(data, 0) >= 0 { + return false + } + text := strings.TrimPrefix(string(data), "\xef\xbb\xbf") + section, haveSection := "", false + var subsection *string + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSuffix(line, "\r") + if strings.ContainsRune(line, '\r') { + return false + } + if gitConfigCommentLn.MatchString(line) { + continue + } + if m := gitConfigHeader.FindStringSubmatch(line); m != nil { + section, haveSection = strings.ToLower(m[1]), true + subsection = nil + if strings.Contains(line[:strings.IndexByte(line, ']')], `"`) { + sub := m[2] + subsection = &sub + } + continue + } + m := gitConfigKeyLine.FindStringSubmatchIndex(line) + if m == nil || !haveSection { + return false + } + e := gitConfigEntry{section: section, subsection: subsection, key: strings.ToLower(line[m[2]:m[3]])} + if m[4] >= 0 { + e.hasValue = true + e.rawValue = line[m[4]:m[5]] + value, ok := parseGitConfigValue(e.rawValue) + if !ok { + return false + } + e.value = value + } + if !visit(e) { + return false + } + } + return true +} + +// parseGitConfigValue strips a trailing comment, surrounding whitespace and +// balanced double quotes from a value it fully understands. +func parseGitConfigValue(raw string) (string, bool) { + if strings.ContainsRune(raw, '\\') || strings.Count(raw, `"`)%2 != 0 { + return "", false + } + var b strings.Builder + inQuote := false + for _, c := range raw { + switch { + case c == '"': + inQuote = !inQuote + continue + case !inQuote && (c == '#' || c == ';'): + return strings.TrimSpace(b.String()), true + } + b.WriteRune(c) + } + return strings.TrimSpace(b.String()), true +} diff --git a/internal/git/gitdir.go b/internal/git/gitdir.go index 31019d7b4..133ed8203 100644 --- a/internal/git/gitdir.go +++ b/internal/git/gitdir.go @@ -88,10 +88,36 @@ func canonicalPinPath(p string) string { // initGitContext populates the gitContext with a single git call. // This is called once per process via sync.Once. +// +// The process-wide context is the one every bd invocation resolves at startup +// (beads directory discovery), so it first tries discoverGitInProcess, which +// answers the common layouts — an ordinary repository, a linked worktree, a +// submodule, no repository at all — from the filesystem with the exact output +// `git rev-parse --git-dir --git-common-dir --show-toplevel` would print, and +// declines (falling back to that subprocess) for anything it cannot prove +// equivalent: GIT_DIR-style overrides, repository config that moves the work +// tree, ownership that would engage safe.directory, and so on. func initGitContext() { + if raw, ok := discoverGitInProcess(); ok { + gitCtx = gitContextFromRevParse("", raw) + return + } gitCtx = loadGitContext("", nil) } +// revParseResult is the answer to `git rev-parse --git-dir --git-common-dir +// --show-toplevel`, in Git's own spelling, or notRepo when Git would report +// that the directory is not inside a repository. +type revParseResult struct { + gitDir, commonDir, topLevel string + notRepo bool + notRepoErr error // why, when notRepo; nil means errNotGitRepository +} + +// errNotGitRepository stands in for git's exit status when in-process +// discovery establishes that there is no repository to find. +var errNotGitRepository = errors.New("no .git found in the working directory or any parent") + func loadGitContext(workDir string, env []string) gitContext { var ctx gitContext // Get all three values with a single git call @@ -116,10 +142,29 @@ func loadGitContext(workDir string, env []string) gitContext { ctx.err = fmt.Errorf("unexpected git rev-parse output: got %d lines, expected 3", len(lines)) return ctx } + return gitContextFromRevParse(workDir, revParseResult{ + gitDir: strings.TrimSpace(lines[0]), + commonDir: strings.TrimSpace(lines[1]), + topLevel: strings.TrimSpace(lines[2]), + }) +} - ctx.gitDirRaw = strings.TrimSpace(lines[0]) - commonDirRaw := strings.TrimSpace(lines[1]) - repoRootRaw := strings.TrimSpace(lines[2]) +// gitContextFromRevParse derives the cached context from rev-parse output, +// whichever of git or discoverGitInProcess produced it, so both paths share +// one normalization. +func gitContextFromRevParse(workDir string, raw revParseResult) gitContext { + var ctx gitContext + if raw.notRepo { + reason := raw.notRepoErr + if reason == nil { + reason = errNotGitRepository + } + ctx.err = fmt.Errorf("not a git repository: %w", reason) + return ctx + } + ctx.gitDirRaw = raw.gitDir + commonDirRaw := raw.commonDir + repoRootRaw := raw.topLevel // Convert commonDir to absolute for reliable comparison absCommon, err := absoluteGitPath(workDir, commonDirRaw) From 533f95295ac2583502e12ba65cc00c2e5ba22276 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 01:05:39 +0000 Subject: [PATCH 006/151] =?UTF-8?q?feat(httpclient):=20BatchApplier=20over?= =?UTF-8?q?=20HTTP=20=E2=80=94=20handshake=20cap,=20dep=5Fadd=20lineage=20?= =?UTF-8?q?on=20the=20wire,=20gc=20mappings=20(S5)=20(#7265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(httpclient): carry DepAddItem.HasSpawner/ThreadID over the wire The http leg's batchApply wire shape dropped a dep_add item's HasSpawner and ThreadID fields -- an edit the caller believed landed. Extend the wire body (ApplyDepAddItem.has_spawner/thread_id), gate sending either field on a new additive behavior capability (issues.batchApply.depAddLineage / CapBatchApplyDepAddLineage) so a client refuses locally before dialing an older server instead of silently dropping graph lineage, and wire the storage-layer spawner-id stamping (StampWaitsForSpawnerID) through both the dolt and uow legs that already had the fields but never stamped them. wire_revision and min_client are unchanged; the golden digest and openapi spec are regenerated. Also fixes three generic bijection/vocabulary gates this change was missing: write_ledger_test.go's applyBatch/item/dep_add carriage table, and the literal behaviorCapabilities lists in internal/httpapi/server_test.go and internal/httpclient/wire/vocabulary_test.go. Co-Authored-By: Claude Sonnet 5 * test(httpclient/wire): pin the item-count gate at 356 items Add TestApplyBatchAt356ItemsWithoutTheCapabilityRefusesBeforeDialing, exercising internal/storage/batchfixtures.Shape356's exact item count -- the design's primary measured large-batch-apply shape -- rather than only the floor+1/ceiling+1 boundaries the existing cases pin. The gate is content-agnostic (it never looks past len(body.Items)), so a synthetic plan here is behaviorally identical to a real Shape356 plan for this one check; realistic-content coverage follows in the next commit. Co-Authored-By: Claude Sonnet 5 * test(httpclient): prove a 356-item Shape356 plan commits atomically or leaves zero rows Add two tests against the real served stack using batchfixtures.Shape356's realistic, heterogeneous content (102 creates, 136 blocks edges, 102 tracks-to-root edges, 16 assign updates): - TestServedBatchApplyShape356CommitsAtomically applies the full plan once and verifies every create landed with a bound key, matching the issues table row-for-row. - TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows appends a 357th item guarding the pre-existing root issue with a deliberately stale ExpectedVersion. Because the root is untouched by the other 356 items, this exercises the real server-side precondition_failed round trip (rather than the static same-request-row ErrValidation rule), asserting errors.Is(err, issueops.ErrVersionMismatch) and that none of the 356 creates landed -- the "never chunk" ceiling: an all-or-nothing request that fails on its last item must leave nothing behind. Co-Authored-By: Claude Sonnet 5 * test(httpclient): close S5 review HIGH findings (lineage gate, thread_id, gc mappings) 2026-10 Opus review on the S5 batch-apply slice found the wire change correct but coverage missing: - HIGH-1: refuseUnservedDepAddLineage (batchapplier.go) had no coverage at all -- deleting it passed every existing test. Add a test pinning that HasSpawner or ThreadID on a server without issues.batchApply.depAddLineage refuses *storage.ErrUnsupported before dialing, and that a plan touching neither field still dials. - HIGH-2: thread_id was never asserted end to end. Add RunBatchApplyCarriesThreadIDOntoTheStoredEdge to the shared conformance package, wire it onto the dolt, embeddeddolt and uow legs, and add the http leg's own case (TestServedBatchApplyCarriesThreadIDOntoTheStoredEdge), which is the one case that proves the field survives both the client's encode and the server's decode at once. - HIGH-3: DESIGN Sec 3.8 ("Guarded writes: no new roles") documents two gc mappings onto BatchApplier that had no coverage: an update carrying a metadata patch plus ExpectedVersion followed by a close in one request (CloseWithMetadataIfMatch, atomic -- a stale version leaves zero changes), and a keyed create/MetadataRefs/dep_add (parent-child included)/trailing-update plan (ApplyGraphPlan). Document both compositions on the BatchApplier interface and add http-leg tests for each. Mutation-verified: removing the lineage gate call, dropping thread_id on the client's encode, and dropping it on the server's decode each now fail a test that previously passed unconditionally. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): nil-guard snapshot consumers, fix spec wording, sync Bazel 2026-10 Opus review MED/LOW findings on the S5 batch-apply slice: - LOW-6: Store.snapshot returns (nil, nil) when a Store has neither a wire nor a cached handshake. Store.Counter() does not call roleWire first (unlike Sweeper/BatchApplier), so this is reachable through the ordinary public accessor and a scoped count would panic on snap.Capabilities. Nil-guard every snapshot() consumer (batchapplier.go, counter.go, sweeper.go) so a nil snapshot is treated as "nothing advertised" and falls through to the capability refusal. Add TestCounterRefusesScopeGracefullyWithNoTransportAndNoSnapshot, mutation-verified to panic without the guard. - LOW-7: the ApplyDepAddItem spec comment said an old server would "silently ignore" has_spawner/thread_id; it answers its own 400 for an unknown member. Fix the wording and regenerate apigen/types.gen.go. - MED-4: internal/httpclient/BUILD.bazel was missing served_batch_apply_large_test.go, expected_revision_contract_test.go, served_dualwrite_history_test.go and the //internal/storage/batchfixtures dep. Sync with gazelle (confirmed idempotent) and go_srcs.py --check. Co-Authored-By: Claude Sonnet 5 * fix(storage): carry thread_id on a same-type dep re-add; bound batch thread_id A same-type re-add of an existing dependency compared and rewrote only metadata, so on both write legs (issueops.addDependencyInTx and the uow dependency repository) a re-add naming a new ThreadID was read as change-free and the thread was dropped, and a metadata-plus-thread re-add stored the metadata without the thread. Both arms now read the stored thread_id through one shared rule, issueops.ReAddedDependencyThread: a named thread the row does not carry is a change (UPDATE, replacement-edge journal entry, exactly one version, #5898 leg 2), and a re-add naming no thread keeps the stored one, so callers that never carry a thread cannot erase it. The journal's EventDep has no thread member, so a thread-only re-add journals the same payload as the original add; the source's version snapshot is what records the new thread, and the comments on both arms now say so instead of promising a complete replacement edge. planApplyBatchDepAdd bounds DepAddItem.ThreadID with CheckFieldLen, so an over-long thread id is ErrValidation before any database work on every leg, as the http leg's decode already refuses it, instead of an opaque mid-transaction "string too long" on the local legs. Only the length bound is shared: the http leg's control-character refusal stays http-only. The HasSpawner/ThreadID docs now name the producer that actually sets HasSpawner (an embedder lowering a graph plan, e.g. gc's ApplyGraphPlan mapping) instead of bd create --graph, and drop the CHANGELOG reference that had no entry behind it. New conformance cases, wired on dolt, embeddeddolt, uow and served: RunBatchApplyCarriesThreadIDOntoAnExistingEdge, RunBatchApplyRefusesAThreadIDLongerThanItsColumn and RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * ci(shards): add the re-add thread test to the embedded storage shard manifest Co-Authored-By: Claude Opus 5.5 (1M context) * ci: trigger the first PR CI run now that the base is main Co-Authored-By: Claude Opus 5.5 (1M context) * fix(httpclient): gate has_spawner to waits-for edges; refuse empty thread_id Review iteration 1 fix-up for #7265 (synthesis F1, F2, F4, F5, F6, F7). F2: the client's CapBatchApplyDepAddLineage gate refused HasSpawner on every edge type, but the role reads the flag on a waits-for edge only (StampWaitsForSpawnerID is a no-op off one). An embedder plan setting it on, say, a blocks edge works against every existing server, yet this client refused it during a mixed-version rollout. depAddNamesSpawner now holds the role's own rule: the flag is gated and encoded on a waits-for edge only, and dropped elsewhere, which loses nothing the role would store. ThreadID stays gated on every edge type, since a thread is stored on all of them. The "spawner flag on any other edge type" subtest is restored to an unadvertising server and now also asserts the flag is dropped from the body, and TestApplyBatchRefusesUnservedDepAddLineageBeforeDialing gains a ThreadID-on-blocks case. The W-DepAddItem.HasSpawner ledger row and the regenerated engdocs/design/http-divergence-ledger.md say the same. F5: the document gives thread_id minLength: 1, but the decoder read an empty string as no thread. applyDepAddItem now refuses a present, empty thread_id with 400 invalid_value naming items[i].dep_add.thread_id, as it does an empty type; an absent one still names no thread. New TestApplyBatchRefusesAnEmptyThreadID; the documented-body test now forwards has_spawner and thread_id as well. F1: bd create --graph does set HasSpawner (buildGraphApplyBatchRequest, on the embedded and proxied legs, since #7209), so the DepAddItem doc names both producers again and the conformance doc's CHANGELOG pointer is restored; the [Unreleased] entry behind it is on the base. F4: the CapBatchApplyDepAddLineage comment said an older server would silently drop the members; it answers either one with 400 invalid_argument/unknown_parameter. F6: the create-edge ledger comment said no operation publishes a thread member; batch apply's dep_add item now does. F7: TestServedBatchApplyGcApplyGraphPlan adds a waits-for item carrying HasSpawner and ThreadID and reads the stored row back, pinning metadata.spawner_id to the resolved spawner and the thread on the served leg. F3 (the journal's EventDep carries no thread member) is not addressed here. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Eddie the Engineer --- .../scripts/embedded-storage-test-shards.txt | 1 + backend/conformance/batch_apply_contract.go | 175 +++++++++++- .../conformance/dependency_editor_contract.go | 90 ++++++ backend/conformance/role_bundle_cases.go | 4 + engdocs/design/http-divergence-ledger.md | 6 +- internal/httpapi/apigen/types.gen.go | 12 +- internal/httpapi/batch_apply.go | 24 +- internal/httpapi/batch_apply_test.go | 30 +- internal/httpapi/routes.go | 13 + internal/httpapi/server_test.go | 3 +- internal/httpapi/spec/openapi.v0.yaml | 32 ++- .../httpapi/wireshape/testdata/golden.json | 12 + internal/httpclient/BUILD.bazel | 2 + internal/httpclient/batchapplier.go | 73 ++++- internal/httpclient/batchapplier_test.go | 154 ++++++++-- internal/httpclient/counter.go | 6 +- internal/httpclient/counts_test.go | 43 +++ internal/httpclient/encode/ledger.go | 23 +- .../served_batch_apply_large_test.go | 155 ++++++++++ .../httpclient/served_batch_apply_test.go | 268 +++++++++++++++++- .../httpclient/served_memories_deps_test.go | 6 +- internal/httpclient/sweeper.go | 14 +- internal/httpclient/wire/handshake.go | 10 + internal/httpclient/wire/vocabulary_test.go | 4 +- .../httpclient/wire/writes_batch_cap_test.go | 34 +++ internal/httpclient/write_ledger_test.go | 22 +- internal/storage/batch_apply.go | 10 +- .../storage/dolt/batch_apply_contract_test.go | 15 + .../dolt/dependency_commit_boundary_test.go | 14 +- .../dolt/dependency_editor_contract_test.go | 6 + internal/storage/domain/db/dependency.go | 37 ++- .../db/dependency_null_metadata_test.go | 8 +- .../embeddeddolt/batch_apply_contract_test.go | 9 + .../dependency_editor_contract_test.go | 6 + internal/storage/issueops/dependencies.go | 76 +++-- .../storage/issueops/dependencies_test.go | 49 +++- .../storage/uow/batch_apply_contract_test.go | 9 + .../uow/dependency_editor_contract_test.go | 1 + issueops/batchapplier.go | 45 ++- 39 files changed, 1353 insertions(+), 148 deletions(-) create mode 100644 internal/httpclient/served_batch_apply_large_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 439186a24..96250f443 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -528,6 +528,7 @@ 15 15 TestEmbeddedDependencyEditorAcceptsAnExternalTarget 15 15 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped 15 15 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse +15 15 TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion 15 15 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering 15 15 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers 15 15 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index 5bb9e7056..49ed97a4f 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "reflect" + "strings" "testing" "github.com/steveyegge/beads/internal/types" @@ -1525,10 +1526,11 @@ func RunBatchApplyNormalizesTheWaitsForGate(t *testing.T, ctx context.Context, f // regression for HIGH-2: a waits-for DepAddItem must only acquire // metadata.spawner_id when the caller explicitly named a spawner // (HasSpawner=true, which bd create --graph sets when the plan declares -// edges[].spawner_key/spawner_id). An edge with no named spawner must keep -// its gate-only metadata untouched — stamping it unconditionally from the -// resolved target caused unnecessary rewrite/version churn when gc -// re-applies the same edge (see CHANGELOG.md). +// edges[].spawner_key/spawner_id, as does an embedder lowering such a plan +// onto this role). An edge with no named spawner must keep its gate-only +// metadata untouched — stamping it unconditionally from the resolved target +// caused unnecessary rewrite/version churn when gc re-applies the same edge +// (see CHANGELOG.md). func RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { t.Helper() spawner := fixture.IssuePrefix + "-spawner-named" @@ -1578,6 +1580,157 @@ func RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t *testing.T, ctx context.Context } } +// RunBatchApplyCarriesThreadIDOntoTheStoredEdge pins DepAddItem.ThreadID +// (the 2026-10 Opus-review HIGH-2 finding): a conversation-threading id named +// on a dep_add item must land on the stored dependency row's own thread_id +// column, and an item that names none must store none. ThreadID is a plain +// column read independently of HasSpawner/Metadata, so this is its own case +// rather than a clause folded onto the spawner one above. +// +// Wired onto the http leg this exercises BOTH halves of the wire at once: the +// client's applyDepAddItemBody encode (batchapplier.go) and the server's +// applyDepAddItemMembers decode (internal/httpapi/batch_apply.go) must each +// carry the field, because either one silently dropping it leaves the stored +// column blank and fails the assertion below. +func RunBatchApplyCarriesThreadIDOntoTheStoredEdge(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + source := fixture.IssuePrefix + "-thread-source" + target := fixture.IssuePrefix + "-thread-target" + batchApplySeedIssue(t, ctx, fixture, source, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, target, types.StatusOpen) + + const thread = "thread-conv-1" + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{ + Source: publicops.Ref{ID: source}, Target: publicops.Ref{ID: target}, + Type: publicops.DepBlocks, ThreadID: thread, + }}, + }, + }) + if got := batchApplyEdgeThreadID(t, ctx, fixture, source, target); got != thread { + t.Errorf("stored edge thread_id = %q, want %q", got, thread) + } + + source2 := fixture.IssuePrefix + "-thread-none-source" + target2 := fixture.IssuePrefix + "-thread-none-target" + batchApplySeedIssue(t, ctx, fixture, source2, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, target2, types.StatusOpen) + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + batchApplyDepAdd(publicops.Ref{ID: source2}, publicops.Ref{ID: target2}, publicops.DepBlocks, ""), + }, + }) + if got := batchApplyEdgeThreadID(t, ctx, fixture, source2, target2); got != "" { + t.Errorf("an edge naming no thread must not stamp thread_id, got %q", got) + } +} + +// RunBatchApplyCarriesThreadIDOntoAnExistingEdge pins DepAddItem.ThreadID on +// the re-add arm: a dep_add naming an edge that already exists with the same +// type is the idempotent re-add, and a thread it names still has to reach the +// stored row. That arm once compared and rewrote metadata alone, so a +// thread-only re-add was a silent no-op and a re-add moving both landed the +// metadata and dropped the thread — on the very path gc re-applies its plans +// through. +// +// The steps run on ONE edge, each from the state the step before it left: a +// thread-only re-add puts a thread on an edge that had none, a re-add moving +// the gate and the thread lands both, and a re-add naming no thread keeps the +// stored one, because an empty ThreadID names no thread rather than asking for +// none. +func RunBatchApplyCarriesThreadIDOntoAnExistingEdge(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + source := fixture.IssuePrefix + "-rethread-source" + spawner := fixture.IssuePrefix + "-rethread-spawner" + batchApplySeedIssue(t, ctx, fixture, source, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, spawner, types.StatusOpen) + + add := func(gate, thread string) { + t.Helper() + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{ + Source: publicops.Ref{ID: source}, Target: publicops.Ref{ID: spawner}, + Type: publicops.DepWaitsFor, Metadata: `{"gate":"` + gate + `"}`, ThreadID: thread, + }}, + }, + }) + } + assertEdge := func(step, wantGate, wantThread string) { + t.Helper() + assertBatchApplyEdgeCount(t, ctx, fixture, source, spawner, 1) + stored := batchApplyEdgeMetadata(t, ctx, fixture, source, spawner) + var meta types.WaitsForMeta + if err := json.Unmarshal([]byte(stored), &meta); err != nil { + t.Fatalf("%s: stored waits-for metadata %q is not a gate object: %v", step, stored, err) + } + if meta.Gate != wantGate { + t.Errorf("%s: stored gate = %q (metadata %q), want %q", step, meta.Gate, stored, wantGate) + } + if got := batchApplyEdgeThreadID(t, ctx, fixture, source, spawner); got != wantThread { + t.Errorf("%s: stored edge thread_id = %q, want %q", step, got, wantThread) + } + } + + add(types.WaitsForAllChildren, "") + assertEdge("the first add, naming no thread", types.WaitsForAllChildren, "") + + add(types.WaitsForAllChildren, "thread-conv-1") + assertEdge("a re-add moving the thread alone", types.WaitsForAllChildren, "thread-conv-1") + + add(types.WaitsForAnyChildren, "thread-conv-2") + assertEdge("a re-add moving the gate and the thread", types.WaitsForAnyChildren, "thread-conv-2") + + add(types.WaitsForAnyChildren, "") + assertEdge("a re-add naming no thread", types.WaitsForAnyChildren, "thread-conv-2") +} + +// RunBatchApplyRefusesAThreadIDLongerThanItsColumn pins the bound on +// DepAddItem.ThreadID: one longer than the thread_id column holds is +// ErrValidation and writes no edge, on every leg alike. The http leg's decode +// refused it as a typed 400 while the local legs carried it to the write, where +// it failed as whatever storage error the column raised, mid-transaction — two +// error classes for one request. The shared plan bounds it now, before any +// database work. +// +// The positive half is the same edge with a thread exactly at the bound, which +// lands whole; without it, a role refusing every thread id satisfies the +// refusal. +func RunBatchApplyRefusesAThreadIDLongerThanItsColumn(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + source := fixture.IssuePrefix + "-longthread-source" + target := fixture.IssuePrefix + "-longthread-target" + batchApplySeedIssue(t, ctx, fixture, source, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, target, types.StatusOpen) + + request := func(thread string) publicops.ApplyBatchRequest { + return publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{ + Source: publicops.Ref{ID: source}, Target: publicops.Ref{ID: target}, + Type: publicops.DepBlocks, ThreadID: thread, + }}, + }, + } + } + tooLong := strings.Repeat("t", types.MaxFieldLen+1) + if _, err := fixture.BatchApplier.ApplyBatch(ctx, request(tooLong)); !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("a %d-character thread_id: error = %v, want ErrValidation", len(tooLong), err) + } + assertBatchApplyEdgeCount(t, ctx, fixture, source, target, 0) + + atBound := strings.Repeat("t", types.MaxFieldLen) + batchApplyMust(t, ctx, fixture, request(atBound)) + if got := batchApplyEdgeThreadID(t, ctx, fixture, source, target); got != atBound { + t.Errorf("a %d-character thread_id stored as %d characters, want it whole", len(atBound), len(got)) + } +} + // RunBatchApplySplicesAForwardMetadataRef pins the one exception to the // backward-only rule (CreateItem.MetadataRefs): "IT IS THE ONE PLACE A KEY MAY // REACH FORWARD … Every id is minted before any splice is applied, so the @@ -2241,6 +2394,20 @@ func batchApplyEdgeMetadata(t *testing.T, ctx context.Context, fixture BatchAppl return blob } +// batchApplyEdgeThreadID reads the stored thread_id column of one edge, which +// is where DepAddItem.ThreadID has to be visible — a plain column, independent +// of the type-specific Metadata blob batchApplyEdgeMetadata reads. +func batchApplyEdgeThreadID(t *testing.T, ctx context.Context, fixture BatchApplyFixture, source, target string) string { + t.Helper() + var value string + const query = "SELECT COALESCE(thread_id, '') FROM dependencies WHERE issue_id = ?" + + " AND COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?" + if err := fixture.QueryScalar(ctx, query, []any{source, target}, &value); err != nil { + t.Fatalf("reading edge thread_id for %s -> %s: %v", source, target, err) + } + return value +} + // batchApplyMetadataKey reads the WHOLE metadata blob off a row and reports one // key's raw bytes. The whole blob is read rather than a JSON path extract, // because that is what lets a case see an absent key and a null one as the diff --git a/backend/conformance/dependency_editor_contract.go b/backend/conformance/dependency_editor_contract.go index df2aacb75..e037e9dbe 100644 --- a/backend/conformance/dependency_editor_contract.go +++ b/backend/conformance/dependency_editor_contract.go @@ -531,6 +531,96 @@ func RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *testing.T, } } +// RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion is the +// thread_id twin of +// RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion. The row +// carries a thread as well as metadata, and the source issue's version snapshot +// carries each edge's thread_id, so a same-type re-add naming a thread the row +// does not carry is a real mutation on the same terms as a metadata change: +// the leaf updates the row in place, journals the replacement edge, and mints +// EXACTLY ONE version (#5898 leg 2). The re-add arm once compared metadata +// alone, which read that re-add as change-free and dropped the thread. +// +// The metadata is identical across every write, so the thread is the only +// thing that moves. A last re-add names no thread, and is the change-free +// no-op: an empty thread names none rather than asking for none, which is +// what keeps the callers that never carry a thread from erasing one. +// +// It drives the writes through fixture.AddDependency for the reason the +// metadata case gives: publicops.DependencyEdge carries no thread either. +func RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t *testing.T, ctx context.Context, fixture DependencyEditorFixture) { + t.Helper() + if fixture.AddDependency == nil { + t.Skip("fixture has no AddDependency hook: a thread is not settable through publicops.DependencyEdge") + } + fixture.SetJournalEnabled(true) + t.Cleanup(func() { fixture.SetJournalEnabled(false) }) + fixture.SetVersionedHistoryEnabled(true) + t.Cleanup(func() { fixture.SetVersionedHistoryEnabled(false) }) + source := fixture.IssuePrefix + "-idemthread-source" + target := fixture.IssuePrefix + "-idemthread-target" + seedDependencyEditorIssue(t, ctx, fixture, source) + seedDependencyEditorIssue(t, ctx, fixture, target) + + const metadata = `{"note":"v1"}` + add := func(thread, refusal string) { + t.Helper() + dep := &types.Dependency{IssueID: source, DependsOnID: target, Type: types.DepBlocks, Metadata: metadata, ThreadID: thread} + if err := fixture.AddDependency(ctx, dep, "writer"); err != nil { + t.Fatalf("%s: %v", refusal, err) + } + assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", source, target, string(types.DepBlocks), 1) + assertDependencyEditorEventCount(t, ctx, fixture, "events", source, types.EventDependencyAdded, 1) + } + storedThread := func() string { + t.Helper() + var thread string + if err := fixture.QueryScalar(ctx, + "SELECT COALESCE(thread_id, '') FROM dependencies WHERE issue_id = ? AND COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?", + []any{source, target}, &thread); err != nil { + t.Fatalf("read stored thread_id for %s -> %s: %v", source, target, err) + } + return thread + } + journalCount := func() int { + t.Helper() + var count int + if err := fixture.QueryScalar(ctx, + "SELECT COUNT(*) FROM bd_events_journal WHERE issue_id = ? AND op = ?", + []any{source, string(storeops.EventDepAdd)}, &count); err != nil { + t.Fatalf("count bd_events_journal dep_add rows for %s: %v", source, err) + } + return count + } + + add("", "AddDependency first") + assertDependencyEditorJournalCountIsOne(t, ctx, fixture, source, string(storeops.EventDepAdd)) + + versionsBefore := countDependencyEditorIssueVersions(t, ctx, fixture, source) + add("thread-conv-1", "re-adding the same edge with a new thread refused") + if got := storedThread(); got != "thread-conv-1" { + t.Errorf("stored thread_id = %q, want %q: the re-add must persist the new thread, not keep the old", got, "thread-conv-1") + } + if got := journalCount(); got != 2 { + t.Errorf("bd_events_journal dep_add rows for %s = %d, want 2: the thread-changed re-add must journal its own replacement-edge entry", source, got) + } + if delta := countDependencyEditorIssueVersions(t, ctx, fixture, source) - versionsBefore; delta != 1 { + t.Errorf("issue_versions rows for %s went %d -> %d (delta %d), want a delta of 1: the thread genuinely changed, so the re-add is a real mutation of the source issue and must mint exactly one version (#5898 leg 2)", source, versionsBefore, versionsBefore+delta, delta) + } + + versionsBefore = countDependencyEditorIssueVersions(t, ctx, fixture, source) + add("", "re-adding the same edge naming no thread refused") + if got := storedThread(); got != "thread-conv-1" { + t.Errorf("stored thread_id after a re-add naming no thread = %q, want %q kept", got, "thread-conv-1") + } + if got := journalCount(); got != 2 { + t.Errorf("bd_events_journal dep_add rows for %s = %d after a change-free re-add, want 2 still", source, got) + } + if delta := countDependencyEditorIssueVersions(t, ctx, fixture, source) - versionsBefore; delta != 0 { + t.Errorf("issue_versions rows for %s went %d -> %d (delta %d) on a re-add naming no thread, want a delta of 0: it keeps the stored thread, so nothing changed (#5898 R3)", source, versionsBefore, versionsBefore+delta, delta) + } +} + // RunDependencyEditorRepeatsWithinOneRequestCollapse pins the clause that // answers a request naming ONE PAIR TWICE. The same-type re-add rule is stated // for the pair rather than for the call, so the second occurrence inside a diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 55385d9b7..b74a615a8 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -45,6 +45,9 @@ var roleContractCases = []roleContract{ RunBatchApplyAcceptsAnExternalEdgeTarget, RunBatchApplyNormalizesTheWaitsForGate, RunBatchApplyStampsSpawnerIDOnlyWhenNamed, + RunBatchApplyCarriesThreadIDOntoTheStoredEdge, + RunBatchApplyCarriesThreadIDOntoAnExistingEdge, + RunBatchApplyRefusesAThreadIDLongerThanItsColumn, RunBatchApplySplicesAForwardMetadataRef, RunBatchApplySplicesASelfMetadataRef, RunBatchApplyRefusesAMetadataRefNoItemDeclares, @@ -260,6 +263,7 @@ var roleContractCases = []roleContract{ RunDependencyEditorSameTypeReAddIsIdempotent, RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion, RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp, + RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion, RunDependencyEditorRepeatsWithinOneRequestCollapse, RunDependencyEditorAttributesItsEventsToTheActor, RunDependencyEditorRetypeRefusalLeavesTheOriginalEdge, diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md index 78bbb101b..9815c30b1 100644 --- a/engdocs/design/http-divergence-ledger.md +++ b/engdocs/design/http-divergence-ledger.md @@ -47,7 +47,7 @@ the behavior. ## Summary -264 rows: 207 refuse, 33 degrade, 24 retired. +264 rows: 205 refuse, 33 degrade, 26 retired. ## Whole-behavior divergences (D9 ledger) @@ -134,8 +134,8 @@ Role-request fields on the served writes that the v0 wire publishes no member fo | W-CreateRequest.Issue | refuse | `CreateRequest.Issue` | a create whose Issue populates any member outside the wire's twenty refuses, naming the member | createIssue publishes the whole create VOCABULARY — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue: the creation time (created_at), because a caller-supplied creation time makes the row disagree with the journal entry that records it and re-dating history is what an import is for; and spec_id, await_*, mol_type, wisp_type, work_type, storage_class, source_*, pinned, is_template and the event quartet, which this surface publishes on no operation, read or write. ONE row rather than one per member for W-BatchCreateItem.Issue's reason — the reason is one reason — and exhaustiveness is held by the same REFLECTION over types.Issue, sharing the role-ignored table with the batch so the two operations cannot disagree about what the ROLE drops. Issue.Comments and Issue.Dependencies are not in this population: the role itself refuses them, so a local create fails too and this is validation rather than divergence. created_by is published on no create shape either, but every one of them STAMPS it from the request's actor (internal/httpapi's create.go, batch_create.go and batch_apply.go) — the local front doors' own rule, since the role copies the member rather than stamping it. So a CreatedBy that names the actor, which is what every CLI create sends, arrives as written, and only one naming someone else refuses: the stamp would silently replace it. A server that predates the stamp stores an empty created_by for that one member | D8 row 16 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-CreateItem.Issue | refuse | `CreateItem.Issue` | a create ITEM whose Issue populates any member outside the wire's twenty refuses, naming the member | ApplyCreateItem publishes exactly createIssue's create vocabulary — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue, for the reasons W-CreateRequest.Issue gives in full. It is a row of its OWN rather than a citation of that one because the operations are different: a caller auditing why their PLAN refuses must not be sent to a row about a single create, and the two can diverge the day either vocabulary moves. The partition itself is SHARED — one carried table, one role-ignored table, held against types.Issue by the same reflection — so they cannot disagree about what the wire carries or about what the role drops. Issue.Comments and Issue.Dependencies are not in this population and are refused as validation on both sides: edges in this role are ITEMS, so a create item has nowhere to put one at all. Nor is a CreatedBy that names the request's actor: the server stamps every create item's created_by from the actor (W-CreateRequest.Issue), so that value arrives as written and only a CreatedBy naming someone else refuses | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-ApplyPatch.ParentID | refuse | `IssuePatch.ParentID` | a parent re-hang inside an update ITEM refuses, where the same member serves on updateIssue | ApplyPatchBody deliberately publishes no parent_id, and the absence is this operation's ONE-EDGE-ONE-SPELLING rule rather than a gap: a parent is a dep_add item of type parent-child, so the ORDER of every edge in a plan stays total and there is exactly one place an edge is written. The single patch has no ordering to express and publishes the member directly (W-IssuePatch.ParentID, retired by client wave ga-7i6by), which is why the same Go field is carried by one document and refused by the other. Refusing is not a loss of capability, only of spelling: a plan re-hangs a parent with a dep_add item, which is the atomic replacement in the position the caller declared it | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| W-DepAddItem.HasSpawner | refuse | `DepAddItem.HasSpawner` | a waits-for edge ITEM that names its spawner refuses | ApplyDepAddItem publishes source, target, type and metadata and no spawner member, and the flag asks for a write only the ROLE can make: metadata's spawner_id, stamped from the resolved target once every id in the batch exists, because a target named by key has no id before then. Dropping the flag would store the edge's gate-only metadata, so an edge whose caller named a spawner would carry none. On every other edge type the role ignores the flag and stores the same row either way, so there it is dropped rather than refused. Upstream ask: a spawner member on ApplyDepAddItem, which retires this row | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| W-DepAddItem.ThreadID | refuse | `DepAddItem.ThreadID` | associating an edge ITEM with a discussion thread refuses | ApplyDepAddItem publishes no thread member, and no operation on this surface does: W-CreateDependency.ThreadID refuses the same column on both create edges. Dropping it would store the edge with no thread, so a replies-to edge would lose the conversation it was written for. It is a row of its own rather than a citation of that one for W-CreateItem.Issue's reason: the operation is different | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-DepAddItem.HasSpawner | retired | `DepAddItem.HasSpawner` | a waits-for edge ITEM that names its spawner used to refuse | RETIRED by issues.batchApply.depAddLineage (S5). ApplyDepAddItem now publishes `has_spawner` — the write this row said only the role could make, metadata's spawner_id stamped from the resolved target — gated on the capability rather than dropped: a caller on a server that has not advertised the token refuses locally before the dial (BatchApplier.refuseUnservedDepAddLineage), and a caller on a server that has sends the member exactly as given. The flag is sent, and gated, on a waits-for edge only, the one type the role reads it on. Off a waits-for edge it is the role's no-op, so the client drops it there without loss — the stored row is the same either way — rather than refusing a request the flag cannot change, or sending an older server a member it answers with a 400 | D8 refuse-not-drop (RETIRED) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-DepAddItem.ThreadID | retired | `DepAddItem.ThreadID` | associating an edge ITEM with a discussion thread used to refuse | RETIRED with W-DepAddItem.HasSpawner (S5, issues.batchApply.depAddLineage). ApplyDepAddItem now publishes `thread_id` as a plain column on the stored edge, gated by the same capability and the same pre-dial refusal rather than dropped — a caller naming a thread on an unadvertising server is refused before the dial, never silently stored with no thread | D8 refuse-not-drop (RETIRED) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-IssuePatch.Labels | retired | `IssuePatch.Labels` | RETIRED — the whole ordered label edit crosses: `labels` replaces, `add_labels` adds, `remove_labels` removes | upstream #5510 published add_labels and remove_labels on IssuePatchBody and client wave ga-jpywb emits both, which is the retirement path this row named. The three travel as FLAT siblings where applyBatch nests them under one object, and the server assembles all three into ONE issueops.LabelPatch — so the algebra the role states (replace, then add, then remove; removal wins) is applied server-side and is never this client's to arrange. WHAT THE REFUSAL WAS PROTECTING is worth keeping after the refusal is gone: degrading an incremental edit onto the replace-only member would have meant reading the set, adding to it and writing it back, which silently drops any label another writer added in between. `bd label add` and every agent that tags work concurrently are exactly that caller, so the refusal was right for as long as the members did not exist — and it is the members that retire it, not a change of mind about the read-modify-write | D8 refuse-not-drop, open questions (tracked-refusal roadmap) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-CloseBatchRequest.ClaimNext | refuse | `CloseBatchRequest.ClaimNext` | folding an atomic claim-after-close onto a batch close refuses | batchCloseIssues publishes actor, force, items and session only — no claim_next member exists for this client to encode a ReadyRequest into, and none exists on the response for a claimed issue to decode back out of. Dropping the field silently would answer a caller's claim-after-close with a close that quietly never claimed anything; see W-CreateBatchRequest.Provenance for the same refuse-not-drop reasoning applied to an absent wire member. Deferred: S13 schedules ReadyLister (and with it, the ready-side plumbing a composed claim would need); no slice currently schedules claim_next itself | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-SweepRequest.ProtectLiveDependents | retired | `SweepRequest.ProtectLiveDependents` | protecting live structural dependents during a sweep used to refuse | RETIRED by the S4 wire extension that sends it. The gap was always the DOCUMENT's rather than a decision: the local Sweeper role carries a structural-dependent protection alongside the referenced-citation one, and apigen.SweepRequest had no protect_live_dependents member for this client to narrow onto, so sending `true` and silently dropping it would have run an UNPROTECTED sweep while the caller believed the structural guard was active. The spec now publishes `protect_live_dependents` (and SweepSkips.live_dependent on the response) behind its own behavior-capability token, issues.sweep.liveDependents, because it is a parameter added to an operation that already shipped. The client sends it only once the handshake advertises the token and refuses locally, before the dial, when it does not — see refuseUnservedSweep (sweeper.go) and W-SweepRequest.Limit for the sibling member closed the same way | D8 refuse-not-drop | TestSweepSendsTheS4MembersOnceTheServerAdvertisesThem | diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index f5cecef32..37e13b5f9 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -495,7 +495,12 @@ type ApplyCreateItem struct { // A WAITS-FOR EDGE IS NORMALIZED RATHER THAN STORED AS ASKED. An absent, empty or `{}` `metadata` on a `waits-for` edge is STORED as `{"gate":"all-children"}`, because a stored waits-for row must be self-describing: readers predating the gate's introduction do not default a missing one, so an empty gate is a row those readers get wrong. A metadata that names a gate keeps it, along with the spawner and also-blocks members a caller may carry, and a gate that is neither `all-children` nor `any-children` is a `400`. Nothing else about that member is interpreted. // // THERE IS NO TYPED `waits_for` MEMBER, and that is the shape rather than an omission: every measured caller already carries the gate as metadata, a typed spelling lowers to these same bytes, and the blob carries members a two-field typed member could not express. One spelling, and it is this one. +// +// `has_spawner` and `thread_id` require `issues.batchApply.depAddLineage`: a client naming either on a server that has not advertised the token refuses locally before the dial rather than sending bytes an older server would answer with its own `400` for an unknown member. type ApplyDepAddItem struct { + // HasSpawner Gated by `issues.batchApply.depAddLineage`. Ignored for every `type` but `waits-for`: on a `waits-for` edge, `true` stamps `metadata.spawner_id` with this item's `target` (the id `bd create --graph`'s spawner_key/spawner_id resolved to), so a reader learns which spawned row to watch without re-deriving it from the graph plan. Absent or `false` leaves `metadata` exactly as given. + HasSpawner *bool `json:"has_spawner,omitempty"` + // Metadata One metadata value: ANY JSON value — string, number, boolean, null, array or object — because typed values enter through the explicit JSON metadata path and persist in older rows. It is not a string, and a client must not decode it as one. // // Where a member of this type is OMITTED, the key is absent; where it is present holding `null`, the key exists and holds null. Those are different states and this surface reports both. @@ -515,6 +520,9 @@ type ApplyDepAddItem struct { // A KEY REACHES BACKWARD ONLY where the ref ADDRESSES a row — an `update.target`, a `close.target`, either endpoint of a `dep_add`. The one exception is `create.metadata_refs`, whose values may reach forward or name their own item's key; the operation's description says why. Target Ref `json:"target"` + // ThreadId Gated by `issues.batchApply.depAddLineage`. A plain column on the stored edge, separate from `metadata`'s blob — the graph-plan thread this edge belongs to, for a reader that groups edges by thread without parsing every edge's metadata. + ThreadId *string `json:"thread_id,omitempty"` + // Type The edge type, from the same OPEN vocabulary `Dependency.type` carries: checked for BEING a storable value, never for membership of a known-types list, so a workspace's own type passes. Type string `json:"type"` } @@ -548,6 +556,8 @@ type ApplyItem struct { // A WAITS-FOR EDGE IS NORMALIZED RATHER THAN STORED AS ASKED. An absent, empty or `{}` `metadata` on a `waits-for` edge is STORED as `{"gate":"all-children"}`, because a stored waits-for row must be self-describing: readers predating the gate's introduction do not default a missing one, so an empty gate is a row those readers get wrong. A metadata that names a gate keeps it, along with the spawner and also-blocks members a caller may carry, and a gate that is neither `all-children` nor `any-children` is a `400`. Nothing else about that member is interpreted. // // THERE IS NO TYPED `waits_for` MEMBER, and that is the shape rather than an omission: every measured caller already carries the gate as metadata, a typed spelling lowers to these same bytes, and the blob carries members a two-field typed member could not express. One spelling, and it is this one. + // + // `has_spawner` and `thread_id` require `issues.batchApply.depAddLineage`: a client naming either on a server that has not advertised the token refuses locally before the dial rather than sending bytes an older server would answer with its own `400` for an unknown member. DepAdd *ApplyDepAddItem `json:"dep_add,omitempty"` // Kind Which member below is read. A CLOSED set, unlike a dependency `type`: every value here is a verb this operation implements, and an unknown one is a request the server cannot execute rather than a workspace's own vocabulary. @@ -1037,7 +1047,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, and `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index 16b9d4a63..723ce8615 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -102,7 +102,7 @@ var ( applyLabelPatchMembers = []string{"add", "remove", "replace"} applyMetadataPatchMembers = []string{"merge", "replace", "set", "unset"} applyCloseItemMembers = []string{"expected_version", "force", "reason", "session", "target"} - applyDepAddItemMembers = []string{"metadata", "source", "target", "type"} + applyDepAddItemMembers = []string{"has_spawner", "metadata", "source", "target", "thread_id", "type"} // applyItemKinds is the tag vocabulary, paired with the payload member each // value names. It is one map rather than a switch so the enum, the member @@ -831,6 +831,28 @@ func applyDepAddItem(prefix string, raw map[string]json.RawMessage) (*issueops.D if metadata, present := raw["metadata"]; present { item.Metadata = string(metadata) } + // has_spawner and thread_id are additive members gated client-side by + // issues.batchApply.depAddLineage (routes.go CapBatchApplyDepAddLineage): + // the server always understands them once wired, so there is no + // capability check here — the gate lives entirely in the client that + // decides whether to SEND them against an older server. + hasSpawner, res := applyBoolMember(raw, prefix, "has_spawner") + if res != nil { + return nil, res + } + item.HasSpawner = hasSpawner + threadID, res := applyTextMember(raw, prefix, "thread_id") + if res != nil { + return nil, res + } + // The document's minLength: 1. An ABSENT thread_id already names no thread + // (and a re-add keeps the stored one), so an empty one would only be a + // second spelling of it — refused, as an empty `type` is, rather than read. + if _, present := raw["thread_id"]; present && threadID == "" { + res := InvalidArgument(prefix+"thread_id", ReasonInvalidValue, "`thread_id` must not be empty; omit it to name no thread") + return nil, &res + } + item.ThreadID = threadID return item, nil } diff --git a/internal/httpapi/batch_apply_test.go b/internal/httpapi/batch_apply_test.go index a60c8ea54..824775f21 100644 --- a/internal/httpapi/batch_apply_test.go +++ b/internal/httpapi/batch_apply_test.go @@ -84,7 +84,7 @@ func TestApplyBatchForwardsEveryLevelOfTheDocumentedBody(t *testing.T) { "expected_version":"42"}}, {"kind":"dep_add","dep_add":{ "source":{"key":"root"},"target":{"id":"bd-7"},"type":"waits-for", - "metadata":{"gate":"any-children"}}} + "metadata":{"gate":"any-children"},"has_spawner":true,"thread_id":"th-plan"}} ] }`) if resp.StatusCode != http.StatusOK { @@ -240,6 +240,11 @@ func TestApplyBatchForwardsEveryLevelOfTheDocumentedBody(t *testing.T) { if depAdd.Metadata != `{"gate":"any-children"}` { t.Errorf("dep_add.metadata = %q, want the caller's bytes unaltered", depAdd.Metadata) } + // The lineage members reach the role as sent: issues.batchApply.depAddLineage + // gates whether a CLIENT sends them, and this server always reads them. + if !depAdd.HasSpawner || depAdd.ThreadID != "th-plan" { + t.Errorf("dep_add.has_spawner/thread_id = %v/%q, want true/%q", depAdd.HasSpawner, depAdd.ThreadID, "th-plan") + } } // TestApplyBatchAnswersWithTheLeanResult pins the response side: ids, a @@ -520,6 +525,29 @@ func TestApplyBatchRejectsTheShapesTheDocumentRefuses(t *testing.T) { } } +// TestApplyBatchRefusesAnEmptyThreadID pins the document's minLength: 1 on a +// dep_add item's thread_id. An absent member already names no thread, so an +// empty one is refused by name rather than read as a second spelling of it. +// The client never sends one (setItemString omits an empty value), so only a +// direct-wire caller reaches this. +func TestApplyBatchRefusesAnEmptyThreadID(t *testing.T) { + applier := &roleBatchApplier{} + ts := newApplyBatchServer(t, applier) + + resp := ts.claim(t, batchApplyPath, `{"actor":"alice","items":[{"kind":"dep_add","dep_add":{ + "source":{"id":"a"},"target":{"id":"b"},"type":"blocks","thread_id":""}}]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["param"] != "items[0].dep_add.thread_id" || body["reason"] != "invalid_value" { + t.Errorf("param/reason = %v/%v, want items[0].dep_add.thread_id/invalid_value", body["param"], body["reason"]) + } + if len(applier.requests()) != 0 { + t.Errorf("the role was called for a refused request; nothing may reach it") + } +} + // TestApplyBatchRefusesMoreItemsThanTheRoleAccepts pins the cap at the edge, so // an over-long plan costs no database work. The number is the role's own // constant rather than a second copy of it. diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 2dbd4f291..4c426275a 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -149,6 +149,18 @@ const CapIssuesSweepLiveDependents = "issues.sweep.liveDependents" // before sending the member to a server that may predate it. const CapIssuesSweepLimit = "issues.sweep.limit" +// CapBatchApplyDepAddLineage is the behavior capability that advertises +// ApplyDepAddItem's `has_spawner` and `thread_id` members (batch_apply.go): +// additive fields a `dep_add` item may carry alongside `source`, `target`, +// `type` and `metadata`. Like CapBatchApplyLarge it names members of an +// EXISTING operation (issues.batchApply already has its own per-operation +// token) rather than a route of its own, so it rides the same +// behaviorCapabilities list. An older server predating this token answers +// either member with `400 invalid_argument`/`unknown_parameter`. The client +// MUST check this token before sending either member to a server that may +// predate it; it refuses locally before the dial when the token is absent. +const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -899,6 +911,7 @@ func (r route) specPathOf() string { var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, + CapBatchApplyDepAddLineage, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 514190e91..6f787874c 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -636,7 +636,8 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // same list. Both are spelled literally here to keep this an independent // oracle rather than a second call to the code under test. want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", - "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit") + "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", + "issues.batchApply.depAddLineage") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 784f24f22..456693a41 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -7969,9 +7969,13 @@ components: `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering - `unknown_parameter`, and `issues.sweep.limit`, which announces + `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than - silently answering `unknown_parameter`. The + silently answering `unknown_parameter`, and + `issues.batchApply.depAddLineage`, which announces that a + `dep_add` item on `POST /v0/beads/issues:batchApply` accepts + `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than + a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. @@ -10301,6 +10305,12 @@ components: a typed spelling lowers to these same bytes, and the blob carries members a two-field typed member could not express. One spelling, and it is this one. + + + `has_spawner` and `thread_id` require `issues.batchApply.depAddLineage`: + a client naming either on a server that has not advertised the token + refuses locally before the dial rather than sending bytes an older + server would answer with its own `400` for an unknown member. properties: source: $ref: '#/components/schemas/Ref' @@ -10316,6 +10326,24 @@ components: a known-types list, so a workspace's own type passes. metadata: $ref: '#/components/schemas/MetadataValue' + has_spawner: + type: boolean + description: >- + Gated by `issues.batchApply.depAddLineage`. Ignored for every + `type` but `waits-for`: on a `waits-for` edge, `true` stamps + `metadata.spawner_id` with this item's `target` (the id `bd create + --graph`'s spawner_key/spawner_id resolved to), so a reader learns + which spawned row to watch without re-deriving it from the graph + plan. Absent or `false` leaves `metadata` exactly as given. + thread_id: + type: string + minLength: 1 + maxLength: 255 + description: >- + Gated by `issues.batchApply.depAddLineage`. A plain column on the + stored edge, separate from `metadata`'s blob — the graph-plan + thread this edge belongs to, for a reader that groups edges by + thread without parsing every edge's metadata. ApplyBatchResponse: type: object diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 1dccd173c..1bff0651a 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -264,6 +264,12 @@ "type": "string", "required": true }, + { + "schema": "ApplyDepAddItem", + "member": "has_spawner", + "type": "boolean", + "required": false + }, { "schema": "ApplyDepAddItem", "member": "metadata", @@ -283,6 +289,12 @@ "type": "object", "required": true }, + { + "schema": "ApplyDepAddItem", + "member": "thread_id", + "type": "string", + "required": false + }, { "schema": "ApplyDepAddItem", "member": "type", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 702900172..f6cb1cbee 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -114,6 +114,7 @@ go_test( "required_served_tier_test.go", "roles_graph_cycles_test.go", "roles_test.go", + "served_batch_apply_large_test.go", "served_batch_apply_test.go", "served_batch_close_test.go", "served_batch_getter_test.go", @@ -181,6 +182,7 @@ go_test( "//internal/httpclient/wire", "//internal/storage", "//internal/storage/backends", + "//internal/storage/batchfixtures", "//internal/storage/domain", "//internal/storage/embeddeddolt", "//internal/storage/issueops", diff --git a/internal/httpclient/batchapplier.go b/internal/httpclient/batchapplier.go index dae4ed8f0..b39c12986 100644 --- a/internal/httpclient/batchapplier.go +++ b/internal/httpclient/batchapplier.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "reflect" + "slices" "strings" "github.com/steveyegge/beads/internal/httpapi/apigen" @@ -84,6 +85,9 @@ func (b *httpBatchApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBat return issueops.ApplyBatchResult{}, invalid( "a batch apply carries %d items; the limit is %d per request", len(req.Items), issueops.MaxApplyBatchItems) } + if err := b.refuseUnservedDepAddLineage(ctx, req); err != nil { + return issueops.ApplyBatchResult{}, err + } body, err := applyBatchBody(req) if err != nil { @@ -96,6 +100,41 @@ func (b *httpBatchApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBat return decodeApplyBatchResult(req, resp) } +// refuseUnservedDepAddLineage scans the whole plan once, before the dial, for +// a dep_add item naming ThreadID, or HasSpawner on a waits-for edge (the one +// type the role reads it on; see depAddNamesSpawner). Neither member is served +// unless the handshake snapshot advertises wire.CapBatchApplyDepAddLineage: +// an older server has never heard of either and would answer them with its +// generic unknown-member 400 at best, or (once a server DOES know the +// members but this gate were skipped) silently honor a field the caller +// never meant to send past an unupgraded fleet. So this checks the +// capability once for the whole request — never per item, and never after +// the dial. +func (b *httpBatchApplier) refuseUnservedDepAddLineage(ctx context.Context, req issueops.ApplyBatchRequest) error { + carriesLineage := false + for _, item := range req.Items { + if item.Kind == issueops.ItemDepAdd && depAddCarriesLineage(item.DepAdd) { + carriesLineage = true + break + } + } + if !carriesLineage { + return nil + } + snap, err := b.store.snapshot(ctx) + if err != nil { + return err + } + // snap is nil, nil whenever Store.snapshot has no transport AND no cached + // handshake (a Store built with a nil wire): nothing was ever advertised, + // so this falls straight through to the refusal below rather than + // dereferencing a nil *apigen.ContextResponse. + if snap != nil && slices.Contains(snap.Capabilities, wire.CapBatchApplyDepAddLineage) { + return nil + } + return b.store.unsupportedCapability("BatchApplier.ApplyBatch", wire.CapBatchApplyDepAddLineage) +} + // applyBatchBody projects the role's request onto the wire body. func applyBatchBody(req issueops.ApplyBatchRequest) (wire.ApplyBatchRequest, error) { items := make([]wire.ApplyItem, 0, len(req.Items)) @@ -368,15 +407,6 @@ func applyDepAddItemBody(item *issueops.DepAddItem) (*apigen.ApplyDepAddItem, er if item.Type == "" { return nil, invalid("a dep_add item names no edge type") } - // ApplyDepAddItem publishes neither member below, so each refuses where the - // role would store something the wire cannot say. The spawner flag only - // does that on a waits-for edge; the role ignores it on every other type. - switch { - case item.HasSpawner && item.Type == issueops.DepWaitsFor: - return nil, refuse(encode.OpApplyBatch, "W-DepAddItem.HasSpawner") - case item.ThreadID != "": - return nil, refuse(encode.OpApplyBatch, "W-DepAddItem.ThreadID") - } out := &apigen.ApplyDepAddItem{Source: source, Target: target, Type: string(item.Type)} // A BLANK blob is ABSENT, not malformed, and reading it any other way is a // real bug rather than strictness: the role's own rule is that an absent, @@ -390,9 +420,34 @@ func applyDepAddItemBody(item *issueops.DepAddItem) (*apigen.ApplyDepAddItem, er } out.Metadata = apigen.MetadataValue(blob) } + // HasSpawner and ThreadID are gated by CapBatchApplyDepAddLineage + // (checked in ApplyBatch, before the dial, over the whole request) — + // this projection only encodes what the gate already cleared, which is + // why HasSpawner is sent only where depAddNamesSpawner holds. + setItemBool(&out.HasSpawner, depAddNamesSpawner(item)) + setItemString(&out.ThreadId, item.ThreadID) return out, nil } +// depAddCarriesLineage reports whether a dep_add item names a member +// CapBatchApplyDepAddLineage gates: ThreadID on any edge, HasSpawner only +// where depAddNamesSpawner holds. Used to scan a whole request once before +// the dial, rather than discovering the gap one item at a time after bytes +// already left for the wire. +func depAddCarriesLineage(item *issueops.DepAddItem) bool { + return item != nil && (depAddNamesSpawner(item) || item.ThreadID != "") +} + +// depAddNamesSpawner reports whether a dep_add item's HasSpawner means +// anything: the role reads it on a waits-for edge only and ignores it on +// every other type. Off a waits-for edge the flag is therefore dropped, not +// sent, and never needs the capability — dropping it loses nothing the role +// would store, where refusing it would fail a mixed-version request the +// flag cannot change. +func depAddNamesSpawner(item *issueops.DepAddItem) bool { + return item.HasSpawner && item.Type == issueops.DepWaitsFor +} + // applyRefBody projects one ref, applying the exactly-one rule the schema // cannot state. // diff --git a/internal/httpclient/batchapplier_test.go b/internal/httpclient/batchapplier_test.go index 3607f59a6..a29e38033 100644 --- a/internal/httpclient/batchapplier_test.go +++ b/internal/httpclient/batchapplier_test.go @@ -12,10 +12,24 @@ import ( "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) +// applyRoleWithSnapshot is applyRole's twin for pinning exactly what the +// handshake advertises, the same idiom counts_test.go's +// countRoleWithSnapshot uses: passing a snapshot straight through New rather +// than accepting applyRole's bare stubStore default. +func applyRoleWithSnapshot(t *testing.T, w *stubWire, snap *apigen.ContextResponse) issueops.BatchApplier { + t.Helper() + applier, err := New(testTarget(t), w, snap).BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + return applier +} + // The batch-apply role's unit gates: what the client DECIDES before the dial, // and what it refuses to believe about the answer. // @@ -52,8 +66,12 @@ func applyOneCreate() issueops.ApplyBatchRequest { // The create item's issue vocabulary is EXACTLY the single create's, so the // partition tables are shared rather than copied and this drives the same sweep // against a different operation and a different ledger row. The patch's own -// excluded member — parent_id — is the second half, and the edge item's two — -// the spawner flag and the thread — are the third. +// excluded member — parent_id — is the second half. +// +// The edge item's spawner flag and thread were a third population here before +// issues.batchApply.depAddLineage: both are CARRIED now, gated on the +// capability rather than refused outright — see +// TestApplyBatchRefusesUnservedDepAddLineageBeforeDialing. func TestApplyBatchRefusesEveryMemberTheWireExcludes(t *testing.T) { t.Run("the create item's carried members are the wire's own", func(t *testing.T) { published := bodyMembers(t, reflect.TypeOf(apigen.ApplyCreateItem{})) @@ -164,34 +182,18 @@ func TestApplyBatchRefusesEveryMemberTheWireExcludes(t *testing.T) { } }) - t.Run("the edge item's two unpublished members refuse", func(t *testing.T) { - for _, test := range []struct { - member string - row string - set func(*issueops.DepAddItem) - }{ - {"HasSpawner", "W-DepAddItem.HasSpawner", func(item *issueops.DepAddItem) { item.HasSpawner = true }}, - {"ThreadID", "W-DepAddItem.ThreadID", func(item *issueops.DepAddItem) { item.ThreadID = "th-1" }}, - } { - t.Run(test.member, func(t *testing.T) { - item := &issueops.DepAddItem{ - Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, Type: issueops.DepWaitsFor, - } - test.set(item) - w := &stubWire{} - _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ - Actor: "planner", - Items: []issueops.ApplyItem{{Kind: issueops.ItemDepAdd, DepAdd: item}}, - }) - assertRefusedBy(t, err, test.row) - if len(w.calls) != 0 { - t.Errorf("the refused member reached the wire: %v", w.calls) - } - }) - } - }) + // The edge item's HasSpawner/ThreadID used to refuse here + // (W-DepAddItem.HasSpawner on a waits-for edge, W-DepAddItem.ThreadID on + // any). Both are now CARRIED, gated by issues.batchApply.depAddLineage on + // the same edges — see + // TestApplyBatchRefusesUnservedDepAddLineageBeforeDialing for the pre-dial + // capability refusal this subtest retired in favor of. t.Run("a spawner flag on any other edge type is the role's no-op, and is not refused", func(t *testing.T) { + // applyRole's snapshot advertises no capability at all: the flag is + // gated only where the role reads it, so a blocks edge carrying it + // dials an older server rather than refusing — and drops the flag + // instead of sending a member that server would answer with a 400. w := &stubWire{} _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ Actor: "planner", @@ -202,8 +204,12 @@ func TestApplyBatchRefusesEveryMemberTheWireExcludes(t *testing.T) { if err != nil { t.Fatalf("a spawner flag on a blocks edge refused: %v", err) } - if got := w.lastApply.Items[0].DepAdd; got == nil || got.Type != string(issueops.DepBlocks) { - t.Errorf("the encoded edge = %+v, want the caller's blocks edge", got) + got := w.lastApply.Items[0].DepAdd + if got == nil || got.Type != string(issueops.DepBlocks) { + t.Fatalf("the encoded edge = %+v, want the caller's blocks edge", got) + } + if got.HasSpawner != nil { + t.Errorf("has_spawner = %v on a blocks edge, want it dropped: the role ignores it there", *got.HasSpawner) } }) @@ -997,3 +1003,91 @@ func TestTheApplyLabelPatchIsTheWholeEdit(t *testing.T) { t.Error("the clear invented an add member") } } + +// TestApplyBatchRefusesUnservedDepAddLineageBeforeDialing is the 2026-10 +// Opus-review HIGH-1 finding: refuseUnservedDepAddLineage (batchapplier.go) +// carried no coverage at all — deleting the function and its call site +// passed every test that existed before this one. +// +// A dep_add item naming ThreadID on any edge, or HasSpawner on a waits-for +// edge, dialed against a server that does not advertise +// CapBatchApplyDepAddLineage, must refuse BEFORE dialing with +// *storage.ErrUnsupported naming the capability. A plan that touches neither +// field must still dial normally against the SAME masked server: the gate +// scopes the two members, not the operation. (HasSpawner off a waits-for edge +// is the role's no-op and dials too — see +// TestApplyBatchRefusesEveryMemberTheWireExcludes.) +func TestApplyBatchRefusesUnservedDepAddLineageBeforeDialing(t *testing.T) { + masked := &apigen.ContextResponse{BdVersion: "1.2.3"} // no CapBatchApplyDepAddLineage + depAddPlan := func(item issueops.DepAddItem) issueops.ApplyBatchRequest { + return issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{Kind: issueops.ItemDepAdd, DepAdd: &item}}, + } + } + + for _, tc := range []struct { + name string + item issueops.DepAddItem + }{ + { + name: "HasSpawner", + item: issueops.DepAddItem{ + Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, + Type: "waits-for", HasSpawner: true, + }, + }, + { + name: "ThreadID", + item: issueops.DepAddItem{ + Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, + Type: "waits-for", ThreadID: "t-1", + }, + }, + { + // Unlike HasSpawner, a thread is stored on every edge type. + name: "ThreadID on a blocks edge", + item: issueops.DepAddItem{ + Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, + Type: "blocks", ThreadID: "t-1", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + w := &stubWire{} + _, err := applyRoleWithSnapshot(t, w, masked).ApplyBatch(t.Context(), depAddPlan(tc.item)) + if err == nil { + t.Fatal("ApplyBatch returned no error, want a pre-dial capability refusal") + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != wire.CapBatchApplyDepAddLineage { + t.Errorf("Capability = %q, want %q", unsup.Capability, wire.CapBatchApplyDepAddLineage) + } + if len(w.calls) != 0 { + t.Errorf("dialed %d times, want 0: a pre-dial refusal must never reach the wire", len(w.calls)) + } + }) + } + + t.Run("neither field set still dials", func(t *testing.T) { + dependsOn := "bd-2" + w := &stubWire{applied: &apigen.ApplyBatchResponse{ + Keys: map[string]string{}, + Items: []apigen.ApplyItemResult{ + {Kind: "dep_add", IssueId: "bd-1", DependsOnId: &dependsOn, Changed: true, Revision: "0"}, + }, + }} + _, err := applyRoleWithSnapshot(t, w, masked).ApplyBatch(t.Context(), depAddPlan(issueops.DepAddItem{ + Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, Type: "waits-for", + })) + if err != nil { + t.Fatalf("ApplyBatch with no lineage fields set: %v", err) + } + if len(w.calls) == 0 { + t.Error("the plan never reached the wire, want a dial: the gate scopes HasSpawner/ThreadID, not the operation") + } + }) +} diff --git a/internal/httpclient/counter.go b/internal/httpclient/counter.go index 8fe62e1ea..54b11c696 100644 --- a/internal/httpclient/counter.go +++ b/internal/httpclient/counter.go @@ -119,7 +119,11 @@ func (c httpCounter) refuseUnservedScope(ctx context.Context, op string, req iss if err != nil { return err } - if slices.Contains(snap.Capabilities, wire.CapCountScope) { + // snap is nil, nil whenever Store.snapshot has no transport AND no cached + // handshake (a Store built with a nil wire): nothing was ever advertised, + // so this falls straight through to the refusal below rather than + // dereferencing a nil *apigen.ContextResponse. + if snap != nil && slices.Contains(snap.Capabilities, wire.CapCountScope) { return nil } return c.store.unsupportedCapability(op, wire.CapCountScope) diff --git a/internal/httpclient/counts_test.go b/internal/httpclient/counts_test.go index af3c67793..ddfa84b1d 100644 --- a/internal/httpclient/counts_test.go +++ b/internal/httpclient/counts_test.go @@ -736,3 +736,46 @@ func TestCountScopeRefusesLocallyWhenTheServerLacksTheCapability(t *testing.T) { } }) } + +// TestCounterRefusesScopeGracefullyWithNoTransportAndNoSnapshot is the 2026-10 +// Opus-review LOW-6 finding: Store.snapshot returns (nil, nil) when a Store +// carries neither a wire NOR a cached handshake, and Store.Counter() — unlike +// Sweeper() and BatchApplier() — never calls roleWire first, so a Counter +// built this way is reachable through the ordinary public accessor. Before +// the nil guard in refuseUnservedScope, a scoped count against such a Counter +// paniced on snap.Capabilities rather than refusing. +// +// Passing a nil snapshot through New is the direct, minimal reproduction: +// Store.Counter() builds on whatever New was given, with no roleWire gate to +// go through first. +func TestCounterRefusesScopeGracefullyWithNoTransportAndNoSnapshot(t *testing.T) { + counter, err := New(testTarget(t), nil, nil).Counter() + if err != nil { + t.Fatalf("Counter(): %v", err) + } + + _, err = counter.Count(t.Context(), issueops.CountRequest{ParentID: "bd-1"}) + if err == nil { + t.Fatal("Count returned no error, want a graceful capability refusal rather than a panic") + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != wire.CapCountScope { + t.Errorf("Capability = %q, want %q", unsup.Capability, wire.CapCountScope) + } + + _, err = counter.CountByGroup(t.Context(), issueops.CountByGroupRequest{ + Filter: issueops.CountRequest{NoParent: true}, GroupBy: issueops.CountGroupStatus, + }) + if err == nil { + t.Fatal("CountByGroup returned no error, want a graceful capability refusal rather than a panic") + } + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != wire.CapCountScope { + t.Errorf("Capability = %q, want %q", unsup.Capability, wire.CapCountScope) + } +} diff --git a/internal/httpclient/encode/ledger.go b/internal/httpclient/encode/ledger.go index 605226c27..f40002da3 100644 --- a/internal/httpclient/encode/ledger.go +++ b/internal/httpclient/encode/ledger.go @@ -759,7 +759,9 @@ func writeSideRows() []Row { // createIssue's own edge publishes `reverse` and `metadata` (its issue // HAS an id for a target to point back at, which a batch item does // not), so the client sends both there. Only ThreadID is refused on - // both, because no operation publishes a thread member. + // both, because neither create schema publishes a thread member — a + // batch apply's dep_add item does, which retired its own row + // (W-DepAddItem.ThreadID) but not these. createDependencyRow("Reverse", "an edge written from the target back to the new issue refuses ON A BATCH CREATE", "BatchCreateDependency carries target_id and type only; dropping Reverse would write the edge in the OPPOSITE direction from the one asked for, which is a different graph. createIssue's CreateIssueDependency DOES publish it, and the single-create path sends it"), createDependencyRow("Metadata", "typed edge metadata refuses ON A BATCH CREATE", @@ -834,21 +836,20 @@ func writeSideRows() []Row { PinnedBy: applyPin, }, { - ID: "W-DepAddItem.HasSpawner", Kind: KindRefuse, + ID: "W-DepAddItem.HasSpawner", Kind: KindRetired, Type: tyApplyDepAddItem, Field: "HasSpawner", - What: "a waits-for edge ITEM that names its spawner refuses", - Why: "ApplyDepAddItem publishes source, target, type and metadata and no spawner member, and the flag asks for a write only the ROLE can make: metadata's spawner_id, stamped from the resolved target once every id in the batch exists, because a target named by key has no id before then. " + - "Dropping the flag would store the edge's gate-only metadata, so an edge whose caller named a spawner would carry none. On every other edge type the role ignores the flag and stores the same row either way, so there it is dropped rather than refused. Upstream ask: a spawner member on ApplyDepAddItem, which retires this row", - SpecRow: "D8 refuse-not-drop", + What: "a waits-for edge ITEM that names its spawner used to refuse", + Why: "RETIRED by issues.batchApply.depAddLineage (S5). ApplyDepAddItem now publishes `has_spawner` — the write this row said only the role could make, metadata's spawner_id stamped from the resolved target — gated on the capability rather than dropped: a caller on a server that has not advertised the token refuses locally before the dial (BatchApplier.refuseUnservedDepAddLineage), and a caller on a server that has sends the member exactly as given. " + + "The flag is sent, and gated, on a waits-for edge only, the one type the role reads it on. Off a waits-for edge it is the role's no-op, so the client drops it there without loss — the stored row is the same either way — rather than refusing a request the flag cannot change, or sending an older server a member it answers with a 400", + SpecRow: "D8 refuse-not-drop (RETIRED)", PinnedBy: applyPin, }, { - ID: "W-DepAddItem.ThreadID", Kind: KindRefuse, + ID: "W-DepAddItem.ThreadID", Kind: KindRetired, Type: tyApplyDepAddItem, Field: "ThreadID", - What: "associating an edge ITEM with a discussion thread refuses", - Why: "ApplyDepAddItem publishes no thread member, and no operation on this surface does: W-CreateDependency.ThreadID refuses the same column on both create edges. Dropping it would store the edge with no thread, so a replies-to edge would lose the conversation it was written for. " + - "It is a row of its own rather than a citation of that one for W-CreateItem.Issue's reason: the operation is different", - SpecRow: "D8 refuse-not-drop", + What: "associating an edge ITEM with a discussion thread used to refuse", + Why: "RETIRED with W-DepAddItem.HasSpawner (S5, issues.batchApply.depAddLineage). ApplyDepAddItem now publishes `thread_id` as a plain column on the stored edge, gated by the same capability and the same pre-dial refusal rather than dropped — a caller naming a thread on an unadvertising server is refused before the dial, never silently stored with no thread", + SpecRow: "D8 refuse-not-drop (RETIRED)", PinnedBy: applyPin, }, { diff --git a/internal/httpclient/served_batch_apply_large_test.go b/internal/httpclient/served_batch_apply_large_test.go new file mode 100644 index 000000000..424dc41e9 --- /dev/null +++ b/internal/httpclient/served_batch_apply_large_test.go @@ -0,0 +1,155 @@ +//go:build cgo + +// Written fresh for OSS beads S5 (no bd-enterprise source copied). + +package httpclient + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/storage/batchfixtures" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// TestServedBatchApplyShape356CommitsAtomically and +// TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows are the http +// leg's half of slice S5's named trio ("a 356-item plan against a masked +// server refuses pre-dial; against the main server it commits atomically; an +// injected mid-plan failure leaves zero rows"). The masked-server half lives +// in internal/httpclient/wire (TestApplyBatchAt356ItemsWithoutTheCapabilityRefusesBeforeDialing): +// the item-count gate it exercises never looks past len(body.Items), so a +// synthetic plan there is behaviorally identical to a REAL +// batchfixtures.Shape356 plan for that one gate. These two cases are what a +// REALISTIC, heterogeneous 356-item plan (the design's primary measured +// shape: 102 creates with metadata, 136 blocks edges, 102 tracks-to-root +// edges, 16 assign updates) must still do once past that gate: commit every +// row in one transaction, or refuse the whole thing and leave none. +// +// Both drive the real in-process bd serve stack (newServedEnv), which +// unconditionally advertises issues.batchApplyLarge in its static +// behaviorCapabilities -- so neither needs the masked handshake the other +// case already covers; they are about the CONTENT this gate lets through, +// not the gate itself. + +// TestServedBatchApplyShape356CommitsAtomically applies the full 356-item +// shape once and asserts the whole plan actually landed: every create's key +// bound to a minted id, and the issues table holding exactly one row per +// create on top of the pre-seeded root -- proving the client's positional +// result-decoding and key-binding checks (decodeApplyBatchResult, +// checkApplyKeyBinding) hold up at this scale, not only on the handful-of- +// items plans the rest of served_batch_apply_test.go uses. +func TestServedBatchApplyShape356CommitsAtomically(t *testing.T) { + env := newServedEnv(t, "hbalg0") + ctx := t.Context() + + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := env.createIssue(ctx, root, "tester"); err != nil { + t.Fatalf("seed root issue: %v", err) + } + + applier, err := env.subject.BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + + plan := batchfixtures.Shape356("tester", root.ID) + wantCreates := 0 + for _, item := range plan.Items { + if item.Kind == issueops.ItemCreate { + wantCreates++ + } + } + + result, err := applier.ApplyBatch(ctx, plan) + if err != nil { + t.Fatalf("ApplyBatch(%d items): %v", len(plan.Items), err) + } + if len(result.Items) != len(plan.Items) { + t.Fatalf("got %d item results, want %d (one per requested item)", len(result.Items), len(plan.Items)) + } + if len(result.Keys) != wantCreates { + t.Errorf("got %d bound keys, want %d (one per named create)", len(result.Keys), wantCreates) + } + + var createdRows int + if err := env.queryScalar(ctx, "SELECT COUNT(*) FROM issues WHERE id != ?", []any{root.ID}, &createdRows); err != nil { + t.Fatalf("count created rows: %v", err) + } + if createdRows != wantCreates { + t.Errorf("issues table holds %d non-root row(s) after a %d-item apply, want %d (every create must have landed, none dropped or duplicated)", + createdRows, len(plan.Items), wantCreates) + } +} + +// TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows is the "injected +// mid-plan failure" half: the full 356-item shape, plus one more item at the +// tail guarding the PRE-EXISTING root issue with a deliberately stale +// ExpectedVersion. The root is never created or touched by any of the 356 +// shape items, so this guard takes the real precondition_failed round trip +// (not the static "guard on a row this same request already wrote" refusal +// RunBatchApplyRefusesExpectedVersionOnARowAnEarlierItemCreated/Touched pin +// elsewhere) -- proving, at realistic scale, both Requirement 2 (a batch +// item's ExpectedVersion rides the strict revision token and a server-side +// precondition_failed maps back to issueops.ErrVersionMismatch via +// errors.Is) and the "never chunk" ceiling: a 357-item ALL-OR-NOTHING +// request that fails on its last item must leave NONE of the other 356 +// rows behind, which a client that silently split the request into several +// smaller ones could not guarantee. +func TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows(t *testing.T) { + env := newServedEnv(t, "hbalg1") + ctx := t.Context() + + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := env.createIssue(ctx, root, "tester"); err != nil { + t.Fatalf("seed root issue: %v", err) + } + + applier, err := env.subject.BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + + plan := batchfixtures.Shape356("tester", root.ID) + staleVersion := int64(999999) + plan.Items = append(plan.Items, issueops.ApplyItem{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: root.ID}, + Patch: issueops.IssuePatch{Title: issueops.Field[string]{Set: true, Value: "clobbered"}}, + ExpectedVersion: &staleVersion, + }, + }) + lastIndex := len(plan.Items) - 1 + + _, err = applier.ApplyBatch(ctx, plan) + if !errors.Is(err, issueops.ErrVersionMismatch) { + t.Fatalf("a stale ExpectedVersion on a %d-item plan: error = %v, want ErrVersionMismatch", len(plan.Items), err) + } + var itemErr *issueops.ItemError + if !errors.As(err, &itemErr) { + t.Fatalf("error = %v, want the refusal wrapped in an *ItemError naming the item", err) + } + if itemErr.Index != lastIndex || itemErr.IssueID != root.ID { + t.Errorf("ItemError = %#v, want Index %d acting on %s", itemErr, lastIndex, root.ID) + } + + // Nothing from the 356-item shape landed: no creates, and the root's own + // title is untouched by the refused guard. + var createdRows int + if err := env.queryScalar(ctx, "SELECT COUNT(*) FROM issues WHERE id != ?", []any{root.ID}, &createdRows); err != nil { + t.Fatalf("count created rows: %v", err) + } + if createdRows != 0 { + t.Errorf("issues table holds %d non-root row(s) after a refused %d-item apply, want 0 (an all-or-nothing request must leave nothing behind)", + createdRows, len(plan.Items)) + } + var title string + if err := env.queryScalar(ctx, "SELECT title FROM issues WHERE id = ?", []any{root.ID}, &title); err != nil { + t.Fatalf("read back root title: %v", err) + } + if title != root.Title { + t.Errorf("root title = %q after a refused guard, want unchanged %q", title, root.Title) + } +} diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index f6c8a6257..d7fa20172 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -6,6 +6,8 @@ package httpclient import ( + "context" + "encoding/json" "errors" "strings" "testing" @@ -34,9 +36,11 @@ import ( // shared database would turn an id collision between two contracts into a // debugging session. -// applyParkBead owns this family's two parks: L-apply-ref's unparks when the -// wire grows a typed ref-key member on the problem document, and -// W-DepAddItem.HasSpawner's when ApplyDepAddItem publishes a spawner member. +// applyParkBead owns this family's one remaining park: L-apply-ref's unparks +// when the wire grows a typed ref-key member on the problem document. +// W-DepAddItem.HasSpawner's park UNPARKED under issues.batchApply.depAddLineage +// (S5) — ApplyDepAddItem now publishes a spawner member — and +// TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed runs unskipped below. const applyParkBead = "ga-mijra" func newServedBatchApplyFixture(t *testing.T, prefix string) conformance.BatchApplyFixture { @@ -174,18 +178,6 @@ func TestServedBatchApplyNormalizesTheWaitsForGate(t *testing.T) { conformance.RunBatchApplyNormalizesTheWaitsForGate(t, t.Context(), newServedBatchApplyFixture(t, "hba1c")) } -// TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed parks on the edge item's -// spawner flag. The case's second half, an edge that names no spawner, is what -// this wire writes for every waits-for edge; its first half is the item the -// client refuses. -func TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed(t *testing.T) { - skipKnownDivergence(t, "W-DepAddItem.HasSpawner", applyParkBead, - "the case names a spawner on a waits-for edge (DepAddItem.HasSpawner), and ApplyDepAddItem publishes no "+ - "spawner member, so the client refuses the item rather than letting the server store the edge without "+ - "the spawner_id the role would stamp from its resolved target.") - conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, t.Context(), newServedBatchApplyFixture(t, "hba27")) -} - func TestServedBatchApplySplicesAForwardMetadataRef(t *testing.T) { conformance.RunBatchApplySplicesAForwardMetadataRef(t, t.Context(), newServedBatchApplyFixture(t, "hba1d")) } @@ -255,6 +247,43 @@ func TestServedBatchApplyRefusesAnUnusableRequest(t *testing.T) { conformance.RunBatchApplyRefusesAnUnusableRequest(t, t.Context(), newServedBatchApplyFixture(t, "hba26")) } +// TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed wires the dep_add +// HasSpawner/ThreadID lineage conformance case onto the http leg (S5): a +// served default Config always advertises CapBatchApplyDepAddLineage (it is +// in httpapi's unconditional behaviorCapabilities), so this exercises the +// member on the WIRE, round-tripping through refuseUnservedDepAddLineage's +// pre-dial check (which never fires here, since the capability is present) +// and applyDepAddItemBody's encoding, server-side batch_apply.go decode, and +// the role's own StampWaitsForSpawnerID. +func TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed(t *testing.T) { + conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, t.Context(), newServedBatchApplyFixture(t, "hba27")) +} + +// TestServedBatchApplyCarriesThreadIDOntoTheStoredEdge wires +// DepAddItem.ThreadID's round trip onto the http leg (S5, 2026-10 Opus-review +// HIGH-2): this is the ONE case that proves the field survives BOTH sides of +// the wire at once — the client's applyDepAddItemBody encode and the server's +// applyDepAddItemMembers decode — since either one dropping thread_id leaves +// the raw dependencies row's thread_id column blank and fails the assertion. +func TestServedBatchApplyCarriesThreadIDOntoTheStoredEdge(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoTheStoredEdge(t, t.Context(), newServedBatchApplyFixture(t, "hba28")) +} + +// TestServedBatchApplyCarriesThreadIDOntoAnExistingEdge wires the re-add arm of +// the same round trip: a thread named on a dep_add whose edge already exists +// must cross the wire and reach the stored row, not stop at the server's +// change-free check. +func TestServedBatchApplyCarriesThreadIDOntoAnExistingEdge(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoAnExistingEdge(t, t.Context(), newServedBatchApplyFixture(t, "hba29")) +} + +// TestServedBatchApplyRefusesAThreadIDLongerThanItsColumn wires the thread_id +// bound onto the http leg, where the server's decode refuses it before the +// role's plan would; both have to arrive as the same ErrValidation. +func TestServedBatchApplyRefusesAThreadIDLongerThanItsColumn(t *testing.T) { + conformance.RunBatchApplyRefusesAThreadIDLongerThanItsColumn(t, t.Context(), newServedBatchApplyFixture(t, "hba2a")) +} + // TestServedBatchApplyRefusesAMetadataRefWithoutNamingTheKey is the running pin // beside the park above: the same request, driven end to end, asserting // everything the contract asserts EXCEPT the key inside the member — the @@ -455,3 +484,212 @@ func TestServedBatchApplyAnswersEveryItemWithoutASnapshot(t *testing.T) { } } } + +// TestServedBatchApplyGcCloseWithMetadataIfMatch pins the first of DESIGN +// §3.8's two gc mappings (2026-10 Opus-review HIGH-3, no new bd surface): gc's +// RAW-transaction CloseWithMetadataIfMatch maps onto BatchApplier[update{Patch +// Metadata, ExpectedVersion}, close{}] in ONE request — see the BatchApplier +// package doc. +// +// It is atomic because the whole request is: the matching-version half lands +// both the metadata write and the close together, and the stale-version half +// below refuses the update, which refuses the request, so the close beside it +// never runs and NEITHER change lands. +func TestServedBatchApplyGcCloseWithMetadataIfMatch(t *testing.T) { + ctx := t.Context() + fixture := newServedBatchApplyFixture(t, "hbagc0") + + id := fixture.IssuePrefix + "-close-with-metadata" + if err := fixture.CreateIssue(ctx, &types.Issue{ID: id, Title: "closed with a metadata guard", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "apply-seed"); err != nil { + t.Fatalf("seeding %s: %v", id, err) + } + var version int64 + if err := fixture.QueryScalar(ctx, "SELECT row_lock FROM issues WHERE id = ?", []any{id}, &version); err != nil { + t.Fatalf("reading row_lock for %s: %v", id, err) + } + + _, err := fixture.BatchApplier.ApplyBatch(ctx, issueops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []issueops.ApplyItem{ + {Kind: issueops.ItemUpdate, Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: id}, + Patch: issueops.IssuePatch{Metadata: issueops.MetadataPatch{ + Merge: issueops.Field[json.RawMessage]{Set: true, Value: json.RawMessage(`{"closed_via":"gc"}`)}, + }}, + ExpectedVersion: &version, + }}, + {Kind: issueops.ItemClose, Close: &issueops.CloseItem{Target: issueops.Ref{ID: id}}}, + }, + }) + if err != nil { + t.Fatalf("CloseWithMetadataIfMatch with the current version: %v", err) + } + if got := batchApplyColumnDirect(t, ctx, &fixture, "status", id); got != string(types.StatusClosed) { + t.Errorf("status = %q, want %q", got, types.StatusClosed) + } + if got := batchApplyColumnDirect(t, ctx, &fixture, "metadata", id); !strings.Contains(got, "closed_via") { + t.Errorf("metadata = %q, want it to carry the merged key", got) + } + + // The stale half: the same shape, a version that is no longer current. The + // WHOLE request refuses and ZERO rows change — neither the metadata merge + // nor the close. + id2 := fixture.IssuePrefix + "-close-with-metadata-stale" + if err := fixture.CreateIssue(ctx, &types.Issue{ID: id2, Title: "a stale guard leaves zero changes", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "apply-seed"); err != nil { + t.Fatalf("seeding %s: %v", id2, err) + } + var version2 int64 + if err := fixture.QueryScalar(ctx, "SELECT row_lock FROM issues WHERE id = ?", []any{id2}, &version2); err != nil { + t.Fatalf("reading row_lock for %s: %v", id2, err) + } + stale := version2 - 1 + _, err = fixture.BatchApplier.ApplyBatch(ctx, issueops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []issueops.ApplyItem{ + {Kind: issueops.ItemUpdate, Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: id2}, + Patch: issueops.IssuePatch{Metadata: issueops.MetadataPatch{ + Merge: issueops.Field[json.RawMessage]{Set: true, Value: json.RawMessage(`{"closed_via":"gc"}`)}, + }}, + ExpectedVersion: &stale, + }}, + {Kind: issueops.ItemClose, Close: &issueops.CloseItem{Target: issueops.Ref{ID: id2}}}, + }, + }) + if !errors.Is(err, issueops.ErrVersionMismatch) { + t.Fatalf("a stale ExpectedVersion: error = %v, want ErrVersionMismatch", err) + } + if got := batchApplyColumnDirect(t, ctx, &fixture, "status", id2); got == string(types.StatusClosed) { + t.Error("the close landed despite the whole request refusing on the stale guard") + } + if got := batchApplyColumnDirect(t, ctx, &fixture, "metadata", id2); strings.Contains(got, "closed_via") { + t.Error("the metadata merge landed despite the whole request refusing on the stale guard") + } +} + +// batchApplyColumnDirect reads one issues column directly, for the two gc- +// mapping tests above: they are plain package tests rather than +// conformance.Run* cases (the composition is http-leg-only, DESIGN §3.8), so +// they cannot reach the conformance package's unexported batchApplyColumn. +func batchApplyColumnDirect(t *testing.T, ctx context.Context, fixture *conformance.BatchApplyFixture, column, id string) string { + t.Helper() + var value string + //nolint:gosec // G201: column is one of this file's own hardcoded names. + query := "SELECT COALESCE(" + column + ", '') FROM issues WHERE id = ?" + if err := fixture.QueryScalar(ctx, query, []any{id}, &value); err != nil { + t.Fatalf("reading issues.%s for %s: %v", column, id, err) + } + return value +} + +// TestServedBatchApplyGcApplyGraphPlan pins the second of DESIGN §3.8's two gc +// mappings (2026-10 Opus-review HIGH-3): gc's RAW RunInTransaction graph-build +// maps onto one BatchApplier request carrying every item kind the plan needs — +// a keyed create, a MetadataRefs splice, a dep_add edge, a parent-child +// dep_add edge, a waits-for edge naming its spawner by key and carrying a +// thread, and a trailing update for AssignAfterCreate — landing together as +// the one plan a caller composed. +func TestServedBatchApplyGcApplyGraphPlan(t *testing.T) { + ctx := t.Context() + fixture := newServedBatchApplyFixture(t, "hbagc1") + + epic := fixture.IssuePrefix + "-plan-epic" + task := fixture.IssuePrefix + "-plan-task" + blocker := fixture.IssuePrefix + "-plan-blocker" + + result, err := fixture.BatchApplier.ApplyBatch(ctx, issueops.ApplyBatchRequest{ + Actor: "apply-writer", + ForceIDPrefix: true, + Items: []issueops.ApplyItem{ + // A keyed create: the plan's root node. + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ + Key: "epic", + Issue: &issueops.Issue{ID: epic, Title: "the plan's root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + }}, + // A second node whose metadata reaches BACKWARD to the epic's minted + // id via MetadataRefs — the splice CreateItem.MetadataRefs documents. + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ + Key: "task", + Issue: &issueops.Issue{ID: task, Title: "spliced to its epic", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + MetadataRefs: map[string]issueops.Ref{"epic_id": {Key: "epic"}}, + }}, + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ + Key: "blocker", + Issue: &issueops.Issue{ID: blocker, Title: "blocks the task", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + }}, + // A dep_add edge, plain. + {Kind: issueops.ItemDepAdd, DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: "task"}, Target: issueops.Ref{Key: "blocker"}, Type: issueops.DepBlocks, + }}, + // A dep_add edge of type parent-child — "parent-child included", same + // item kind as the one above, a different Type. + {Kind: issueops.ItemDepAdd, DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: "task"}, Target: issueops.Ref{Key: "epic"}, Type: issueops.DepParentChild, + }}, + // A waits-for edge whose spawner is named by plan KEY, with a thread: + // the lineage gc's mapping carries (issues.batchApply.depAddLineage). + // The spawner stamp can only resolve once that key has an id. + {Kind: issueops.ItemDepAdd, DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: "epic"}, Target: issueops.Ref{Key: "blocker"}, Type: issueops.DepWaitsFor, + HasSpawner: true, ThreadID: "th-plan", + }}, + // A trailing update for AssignAfterCreate, resolving the key BACKWARD + // to the row the first items in this same request just minted. + {Kind: issueops.ItemUpdate, Update: &issueops.UpdateItem{ + Target: issueops.Ref{Key: "task"}, + Patch: issueops.IssuePatch{Assignee: issueops.Field[string]{Set: true, Value: "alice"}}, + }}, + }, + }) + if err != nil { + t.Fatalf("ApplyGraphPlan: %v", err) + } + if len(result.Items) != 7 { + t.Fatalf("ApplyBatch(7 items) returned %d item results", len(result.Items)) + } + + var metadata string + if err := fixture.QueryScalar(ctx, "SELECT COALESCE(metadata, '') FROM issues WHERE id = ?", []any{task}, &metadata); err != nil { + t.Fatalf("reading the spliced metadata: %v", err) + } + if !strings.Contains(metadata, result.Keys["epic"]) { + t.Errorf("task metadata = %q, want the epic's minted id %s spliced over epic_id", metadata, result.Keys["epic"]) + } + + var edges int + const edgeQuery = `SELECT COUNT(*) FROM dependencies WHERE issue_id = ? AND type = ? + AND COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?` + if err := fixture.QueryScalar(ctx, edgeQuery, []any{task, "blocks", blocker}, &edges); err != nil { + t.Fatalf("counting the blocks edge: %v", err) + } + if edges != 1 { + t.Errorf("blocks edges task->blocker = %d, want 1", edges) + } + if err := fixture.QueryScalar(ctx, edgeQuery, []any{task, "parent-child", epic}, &edges); err != nil { + t.Fatalf("counting the parent-child edge: %v", err) + } + if edges != 1 { + t.Errorf("parent-child edges task->epic = %d, want 1", edges) + } + + var waitsFor, thread string + const lineageQuery = `SELECT COALESCE(metadata, ''), COALESCE(thread_id, '') FROM dependencies + WHERE issue_id = ? AND type = ? AND COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?` + if err := fixture.QueryScalar(ctx, lineageQuery, []any{epic, "waits-for", blocker}, &waitsFor, &thread); err != nil { + t.Fatalf("reading the waits-for edge epic->blocker: %v", err) + } + var meta types.WaitsForMeta + if err := json.Unmarshal([]byte(waitsFor), &meta); err != nil { + t.Fatalf("waits-for metadata %q is not a gate object: %v", waitsFor, err) + } + if meta.SpawnerID != result.Keys["blocker"] { + t.Errorf("waits-for spawner_id = %q, want the blocker key's minted id %q", meta.SpawnerID, result.Keys["blocker"]) + } + if thread != "th-plan" { + t.Errorf("waits-for thread_id = %q, want %q", thread, "th-plan") + } + + if got := batchApplyColumnDirect(t, ctx, &fixture, "assignee", task); got != "alice" { + t.Errorf("task assignee = %q, want %q (AssignAfterCreate)", got, "alice") + } +} diff --git a/internal/httpclient/served_memories_deps_test.go b/internal/httpclient/served_memories_deps_test.go index 6d97f1d8b..014f38ce3 100644 --- a/internal/httpclient/served_memories_deps_test.go +++ b/internal/httpclient/served_memories_deps_test.go @@ -416,7 +416,7 @@ func TestServedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate(t *testi conformance.RunDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate(t, t.Context(), newServedDependencyEditorFixture(t, "hd38")) } -// The two same-type re-add cases, newly wired: both drive the re-add through +// The same-type re-add cases, newly wired: each drives the re-add through // fixture.AddDependency (the reference store's own raw path, same reason // ClosedChildAddSatisfiesAnAnyChildrenGate reaches past the role above) and // need dual-write version history on, which the fixture now wires onto the @@ -429,3 +429,7 @@ func TestServedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion(t func TestServedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *testing.T) { conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t, t.Context(), newServedDependencyEditorFixture(t, "hd40")) } + +func TestServedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t *testing.T) { + conformance.RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t, t.Context(), newServedDependencyEditorFixture(t, "hd41")) +} diff --git a/internal/httpclient/sweeper.go b/internal/httpclient/sweeper.go index 28b0e9785..f9c2a7a00 100644 --- a/internal/httpclient/sweeper.go +++ b/internal/httpclient/sweeper.go @@ -135,13 +135,21 @@ func (s *httpSweeper) refuseUnservedSweep(ctx context.Context, tier apigen.Sweep if err != nil { return err } - if tier == apigen.WispsPlane && !slices.Contains(snap.Capabilities, wire.CapSweepWispsPlane) { + // snap is nil, nil whenever Store.snapshot has no transport AND no cached + // handshake (a Store built with a nil wire): nothing was ever advertised, + // so every check below reads a nil capability list rather than + // dereferencing a nil *apigen.ContextResponse. + var capabilities []string + if snap != nil { + capabilities = snap.Capabilities + } + if tier == apigen.WispsPlane && !slices.Contains(capabilities, wire.CapSweepWispsPlane) { return s.store.unsupportedCapability("Sweeper.Sweep", wire.CapSweepWispsPlane) } - if req.ProtectLiveDependents && !slices.Contains(snap.Capabilities, wire.CapSweepLiveDependents) { + if req.ProtectLiveDependents && !slices.Contains(capabilities, wire.CapSweepLiveDependents) { return s.store.unsupportedCapability("Sweeper.Sweep", wire.CapSweepLiveDependents) } - if req.Limit != 0 && !slices.Contains(snap.Capabilities, wire.CapSweepLimit) { + if req.Limit != 0 && !slices.Contains(capabilities, wire.CapSweepLimit) { return s.store.unsupportedCapability("Sweeper.Sweep", wire.CapSweepLimit) } return nil diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 13b10b586..b7db819c2 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -195,6 +195,15 @@ const CapSweepLiveDependents = "issues.sweep.liveDependents" // of what used to be a permanent refusal. const CapSweepLimit = "issues.sweep.limit" +// CapBatchApplyDepAddLineage is the behavior capability announcing that +// issues.batchApply's dep_add items accept `has_spawner` and `thread_id`, +// spelled exactly as httpapi's constant of the same name (held to it by +// TestCapabilityTableMatchesTheServerRouteTable). batchapplier.go reads this +// token before sending either field, refusing locally with a typed +// capability error rather than silently dropping graph lineage a caller +// asked to carry against an older server. +const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" + // CapExternalDependencies is the CONDITIONAL behavior capability announcing // that the ready, claim and close operations of this server apply bd's // external-dependency policy themselves, spelled exactly as httpapi's constant @@ -335,6 +344,7 @@ func (e *WireRevisionSkewError) Unwrap() error { return ErrWireRevisionSkew } var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, + CapBatchApplyDepAddLineage, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 1bee3f45c..23d98908d 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -195,9 +195,11 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if !slices.Equal(behaviorCapabilities, []string{ httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, + httpapi.CapBatchApplyDepAddLineage, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ - "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents and issues.sweep.limit tokens", behaviorCapabilities) + "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit and "+ + "issues.batchApply.depAddLineage tokens", behaviorCapabilities) } } diff --git a/internal/httpclient/wire/writes_batch_cap_test.go b/internal/httpclient/wire/writes_batch_cap_test.go index c023b090b..c3d6d4a5e 100644 --- a/internal/httpclient/wire/writes_batch_cap_test.go +++ b/internal/httpclient/wire/writes_batch_cap_test.go @@ -97,6 +97,40 @@ func TestApplyBatchBetweenTheFloorAndTheCeilingWithTheCapabilitySucceeds(t *test } } +// TestApplyBatchAt356ItemsWithoutTheCapabilityRefusesBeforeDialing exercises +// the item-count gate at exactly 356 items -- the design's primary measured +// large-batch-apply shape (internal/storage/batchfixtures.Shape356) -- rather +// than only at the floor+1/ceiling+1 boundaries the other cases above pin. +// applyBatchRequestOfSize's own doc comment is why a synthetic 356-item plan +// is behaviorally identical to a real Shape356 plan for THIS gate: the cap +// logic never looks past len(body.Items). What a REALISTIC Shape356 payload +// must still get past this same gate -- and commit atomically once it does -- +// is covered end-to-end against the real served stack by +// internal/httpclient's TestServedBatchApplyShape356CommitsAtomically and +// TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows. +func TestApplyBatchAt356ItemsWithoutTheCapabilityRefusesBeforeDialing(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, serveEverything(contextBody("v0", "proj-1", withBatchApply()...))) + _, err := c.ApplyBatch(ctx(t), applyBatchRequestOfSize(356)) + + var capErr *CapabilityError + if !errors.As(err, &capErr) { + t.Fatalf("err = %v (%T), want *CapabilityError", err, err) + } + if capErr.Capability != CapBatchApplyLarge { + t.Errorf("Capability = %q, want %q", capErr.Capability, CapBatchApplyLarge) + } + // Never dialed past the handshake: a masked server (one that does not + // advertise CapBatchApplyLarge) must refuse a 356-item plan PRE-DIAL, the + // same way it refuses any other over-floor count. + if rec.count() != 1 { + t.Fatalf("made %d requests, want only the handshake (a 356-item plan must refuse pre-dial against a server that does not advertise %s)", + rec.count(), CapBatchApplyLarge) + } + if got := rec.at(t, 0).path; got != PathContext { + t.Errorf("only request = %q, want %q", got, PathContext) + } +} + func TestApplyBatchOverTheAbsoluteCeilingRefusesRegardlessOfTheCapability(t *testing.T) { for _, tc := range []struct { name string diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go index 29967f9c0..945c51477 100644 --- a/internal/httpclient/write_ledger_test.go +++ b/internal/httpclient/write_ledger_test.go @@ -594,18 +594,24 @@ func writeShapes() []writeShape { }, }, { - // The edge item. The gate normalization a waits-for edge gets is - // the ROLE's and travels inside the blob. HasSpawner and ThreadID - // have no wire member, so they are absent from carried below and - // refuse under W-DepAddItem.HasSpawner and W-DepAddItem.ThreadID. + // The edge item. TOTAL. The gate normalization a waits-for edge + // gets is the ROLE's and travels inside the blob. HasSpawner and + // ThreadID are carried unconditionally at this structural layer — + // CapBatchApplyDepAddLineage is a separate, client-side refusal + // (batchapplier.go's refuseUnservedDepAddLineage) gating WHETHER + // this client sends either member against an older server, not + // whether the field reaches the wire body type at all. (HasSpawner + // off a waits-for edge is the role's no-op, dropped by value.) name: "applyBatch/item/dep_add", source: reflect.TypeOf(issueops.DepAddItem{}), body: reflect.TypeOf(apigen.ApplyDepAddItem{}), carried: map[string]carriage{ - "Source": member("source"), - "Target": member("target"), - "Type": member("type"), - "Metadata": member("metadata"), + "Source": member("source"), + "Target": member("target"), + "Type": member("type"), + "Metadata": member("metadata"), + "HasSpawner": member("has_spawner"), + "ThreadID": member("thread_id"), }, }, { diff --git a/internal/storage/batch_apply.go b/internal/storage/batch_apply.go index 1a2d30afb..6df88107b 100644 --- a/internal/storage/batch_apply.go +++ b/internal/storage/batch_apply.go @@ -243,8 +243,11 @@ func planApplyBatchClose(item *issueops.CloseItem, index int, keyIndex map[strin return nil } -// planApplyBatchDepAdd checks an edge's endpoints and type and normalizes its -// gate metadata. +// planApplyBatchDepAdd checks an edge's endpoints, type and thread id and +// normalizes its gate metadata. The thread id gets its column's bound here, the +// one the http leg's decode already applies, so an over-long one is +// ErrValidation before any database work on every leg rather than a storage +// error mid-transaction on the local ones. // // IT RECORDS NOTHING AS TOUCHED, and that is a decision rather than an // oversight. The ExpectedVersion rule above refuses a guard on a row this @@ -269,6 +272,9 @@ func planApplyBatchDepAdd(item *issueops.DepAddItem, index int, keyIndex map[str return fmt.Errorf("%w: apply batch item %d requires a dependency type (max %d chars)", issueops.ErrValidation, index, types.MaxDependencyTypeLen) } + if err := types.CheckFieldLen("dependency thread_id", item.ThreadID); err != nil { + return fmt.Errorf("%w: apply batch item %d: %v", issueops.ErrValidation, index, err) + } metadata, err := normalizeApplyEdgeMetadata(item.Type, item.Metadata) if err != nil { return fmt.Errorf("%w: apply batch item %d: %v", issueops.ErrValidation, index, err) diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index 7c32d9a37..13b950360 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -180,6 +180,21 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, ctx, fixture) }) + t.Run("CarriesThreadIDOntoTheStoredEdge", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyCarriesThreadIDOntoTheStoredEdge(t, ctx, fixture) + }) + t.Run("CarriesThreadIDOntoAnExistingEdge", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyCarriesThreadIDOntoAnExistingEdge(t, ctx, fixture) + }) + t.Run("RefusesAThreadIDLongerThanItsColumn", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyRefusesAThreadIDLongerThanItsColumn(t, ctx, fixture) + }) t.Run("SplicesAForwardMetadataRef", func(t *testing.T) { ctx, cancel := testContext(t) defer cancel() diff --git a/internal/storage/dolt/dependency_commit_boundary_test.go b/internal/storage/dolt/dependency_commit_boundary_test.go index 19421828b..b952fd544 100644 --- a/internal/storage/dolt/dependency_commit_boundary_test.go +++ b/internal/storage/dolt/dependency_commit_boundary_test.go @@ -68,9 +68,21 @@ func (c *dependencyCommitBoundaryConn) QueryContext(_ context.Context, query str return &dependencyCommitBoundaryRows{columns: []string{"exists"}}, nil case strings.Contains(query, "SELECT issue_type FROM wisps WHERE id = ?"): return &dependencyCommitBoundaryRows{columns: []string{"issue_type"}, values: [][]driver.Value{{"task"}}}, nil + // The add path's existence probe reads the stored edge's metadata and + // thread_id alongside its type: a same-type re-add compares both to decide + // whether it changes anything. + case strings.Contains(query, "SELECT type, metadata, thread_id FROM dependencies"), + strings.Contains(query, "SELECT type, metadata, thread_id FROM wisp_dependencies"): + if c.driver.newEdge { + return &dependencyCommitBoundaryRows{columns: []string{"type", "metadata", "thread_id"}}, nil + } + return &dependencyCommitBoundaryRows{ + columns: []string{"type", "metadata", "thread_id"}, + values: [][]driver.Value{{"related", "{}", nil}}, + }, nil // The remove path reads the edge's metadata alongside its type: the events // journal records the removed edge's full payload so a consumer can replay - // it (bd-opisf). The add path's existence probe still selects type alone. + // it (bd-opisf). case strings.Contains(query, "SELECT type, metadata FROM dependencies"), strings.Contains(query, "SELECT type, metadata FROM wisp_dependencies"): if c.driver.newEdge { diff --git a/internal/storage/dolt/dependency_editor_contract_test.go b/internal/storage/dolt/dependency_editor_contract_test.go index 3f3b7d26a..f5fac1515 100644 --- a/internal/storage/dolt/dependency_editor_contract_test.go +++ b/internal/storage/dolt/dependency_editor_contract_test.go @@ -55,6 +55,12 @@ func TestDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *testing.T) conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t, ctx, fixture) } +func TestDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t *testing.T) { + fixture, ctx, cleanup := newDoltDependencyEditorFixture(t, "idemthread") + defer cleanup() + conformance.RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t, ctx, fixture) +} + func TestDependencyEditorRepeatsWithinOneRequestCollapse(t *testing.T) { fixture, ctx, cleanup := newDoltDependencyEditorFixture(t, "repeat") defer cleanup() diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index 066ed7bd5..dc4332bb1 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -110,12 +110,12 @@ func (r *dependencySQLRepositoryImpl) Insert(ctx context.Context, dep *types.Dep table := pickDepTable(opts.UseWispsTable) var existingType string - var existingMetadataNS sql.NullString + var existingMetadataNS, existingThreadNS sql.NullString err := r.runner.QueryRowContext(ctx, //nolint:gosec // G201: table and depTargetExpr are hardcoded constants - fmt.Sprintf("SELECT type, metadata FROM %s WHERE issue_id = ? AND %s = ?", table, depTargetExpr), + fmt.Sprintf("SELECT type, metadata, thread_id FROM %s WHERE issue_id = ? AND %s = ?", table, depTargetExpr), dep.IssueID, dep.DependsOnID, - ).Scan(&existingType, &existingMetadataNS) + ).Scan(&existingType, &existingMetadataNS, &existingThreadNS) switch { case err == nil: existingMetadata := existingMetadataNS.String @@ -123,28 +123,35 @@ func (r *dependencySQLRepositoryImpl) Insert(ctx context.Context, dep *types.Dep existingMetadata = "{}" } if existingType == string(dep.Type) { - if issueops.DependencyMetadataEqual(existingMetadata, metadata) { - // Same type, same metadata: a change-free write. Nothing is - // written and nothing is journaled (#5898 R3). + // A re-add naming a thread the row does not carry moves it; one + // naming none keeps the stored thread (the issueops twin's rule). + thread, threadChanged := issueops.ReAddedDependencyThread(existingThreadNS, dep.ThreadID) + if !threadChanged && issueops.DependencyMetadataEqual(existingMetadata, metadata) { + // Same type, same metadata, same thread: a change-free write. + // Nothing is written and nothing is journaled (#5898 R3). return nil } //nolint:gosec // G201: table and depTargetExpr are hardcoded constants if _, err := r.runner.ExecContext(ctx, - fmt.Sprintf("UPDATE %s SET metadata = ? WHERE issue_id = ? AND %s = ?", table, depTargetExpr), - metadata, dep.IssueID, dep.DependsOnID, + fmt.Sprintf("UPDATE %s SET metadata = ?, thread_id = ? WHERE issue_id = ? AND %s = ?", table, depTargetExpr), + metadata, thread, dep.IssueID, dep.DependsOnID, ); err != nil { return fmt.Errorf("db: DependencySQLRepository.Insert: refresh metadata: %w", err) } - // A same-type add refreshes edge metadata. It is an observable graph - // mutation, so emit the complete replacement edge for replay. + // A same-type add refreshes the edge's metadata or thread. It is an + // observable graph mutation, so journal the replacement edge for + // replay. EventDep has no thread member, so a thread-only change + // journals the same payload as the original add; the source's + // version snapshot is what records the new thread. if err := issueops.RecordDepEventInTx(ctx, r.runner, issueops.EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor); err != nil { return err } - // The metadata genuinely changed, so this re-add is a real - // durable-state mutation of the source issue and mints on the - // same terms as a new edge (#5898 leg 2: "a same-type re-add - // whose metadata actually changed mints EXACTLY ONE version - // carrying the new state"). + // The metadata or thread genuinely changed, so this re-add is a + // real durable-state mutation of the source issue (its version + // snapshot carries each edge's thread_id) and mints on the same + // terms as a new edge (#5898 leg 2: "a same-type re-add whose + // metadata actually changed mints EXACTLY ONE version carrying the + // new state"). return issueops.RecordVersionInTx(ctx, r.runner, dep.IssueID, actor) } return &domain.DependencyTypeConflictError{ diff --git a/internal/storage/domain/db/dependency_null_metadata_test.go b/internal/storage/domain/db/dependency_null_metadata_test.go index 60f8a68cd..e15cbead6 100644 --- a/internal/storage/domain/db/dependency_null_metadata_test.go +++ b/internal/storage/domain/db/dependency_null_metadata_test.go @@ -29,7 +29,9 @@ func newMockDependencyRepo(t *testing.T) (sqlmock.Sqlmock, *dependencySQLReposit // which would turn every subsequent re-add of that edge into an error on a // path that used to be idempotently happy. NULL is the absent-metadata state, // so it reads as `{}` and the change-free re-add of a metadata-free edge stays -// the no-op it was. +// the no-op it was. thread_id is nullable the same way and read by the same +// gate; a re-add naming no thread keeps a NULL one, so the row holds NULL in +// both columns and the re-add is still change-free. // // The repair landed on both planes; without this the two planes are pinned // asymmetrically and a revert of the serve-mode scan alone has nothing red to @@ -37,9 +39,9 @@ func newMockDependencyRepo(t *testing.T) (sqlmock.Sqlmock, *dependencySQLReposit func TestDependencyInsertReadsNullStoredMetadataAsEmptyObject(t *testing.T) { mock, repo := newMockDependencyRepo(t) - mock.ExpectQuery(regexp.QuoteMeta("SELECT type, metadata FROM dependencies")). + mock.ExpectQuery(regexp.QuoteMeta("SELECT type, metadata, thread_id FROM dependencies")). WithArgs("dep-a", "dep-b"). - WillReturnRows(sqlmock.NewRows([]string{"type", "metadata"}).AddRow(string(types.DepRelated), nil)) + WillReturnRows(sqlmock.NewRows([]string{"type", "metadata", "thread_id"}).AddRow(string(types.DepRelated), nil, nil)) dep := &types.Dependency{IssueID: "dep-a", DependsOnID: "dep-b", Type: types.DepRelated} // DepRelated is not a scheduling edge and the hierarchy check is declared diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index 3ed6724a9..c7be4ca2d 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -116,6 +116,15 @@ func TestBatchApplyContract(t *testing.T) { t.Run("StampsSpawnerIDOnlyWhenNamed", func(t *testing.T) { conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, ctx, fixture) }) + t.Run("CarriesThreadIDOntoTheStoredEdge", func(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoTheStoredEdge(t, ctx, fixture) + }) + t.Run("CarriesThreadIDOntoAnExistingEdge", func(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoAnExistingEdge(t, ctx, fixture) + }) + t.Run("RefusesAThreadIDLongerThanItsColumn", func(t *testing.T) { + conformance.RunBatchApplyRefusesAThreadIDLongerThanItsColumn(t, ctx, fixture) + }) t.Run("SplicesAForwardMetadataRef", func(t *testing.T) { conformance.RunBatchApplySplicesAForwardMetadataRef(t, ctx, fixture) }) diff --git a/internal/storage/embeddeddolt/dependency_editor_contract_test.go b/internal/storage/embeddeddolt/dependency_editor_contract_test.go index 15fb44f16..88335f2d7 100644 --- a/internal/storage/embeddeddolt/dependency_editor_contract_test.go +++ b/internal/storage/embeddeddolt/dependency_editor_contract_test.go @@ -57,6 +57,12 @@ func TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *te conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "idemmetasame")) } +func TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "idemthread")) +} + func TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse(t *testing.T) { skipUnlessEmbeddedDolt(t) ctx := t.Context() diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index d6acfe91f..17b113438 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -32,6 +32,19 @@ func DependencyMetadataEqual(a, b string) bool { return bytes.Equal(aCanon, bCanon) } +// ReAddedDependencyThread returns the thread_id a same-type re-add leaves on an +// existing edge and whether that differs from the stored one. A named thread +// the row does not already carry replaces it; an empty one names no thread +// rather than asking for none, so the stored value — NULL included — is kept. +// Both dependency re-add idempotency checks share this, alongside +// DependencyMetadataEqual, so they can't drift. +func ReAddedDependencyThread(stored sql.NullString, requested string) (sql.NullString, bool) { + if requested == "" || requested == stored.String { + return stored, false + } + return sql.NullString{String: requested, Valid: true}, true +} + type DepTargetKind int const ( @@ -178,8 +191,9 @@ type DepTargetPrecheck struct { // - Source/target existence validation // - Hierarchy deadlock validation for blocking deps (GH#1495, bd-wg7ve) // - Cycle detection via recursive CTE across both dependency tables -// - Idempotent same-type re-adds: a metadata change updates it, a -// change-free re-add writes and journals nothing (#5898 R3) +// - Idempotent same-type re-adds: a metadata or thread change updates it +// (an empty ThreadID keeps the stored thread), a change-free re-add +// writes and journals nothing (#5898 R3) // - Type conflict detection // // The caller is responsible for transaction lifecycle, dolt commits, and @@ -195,9 +209,9 @@ type DepTargetPrecheck struct { // state, so it mints one version row for the source — on EVERY leg that reaches // this helper (the dependency editor, the legacy store verbs, batch apply), // whether or not an audit event was requested. A same-type re-add that changes -// metadata is the same kind of durable-state change and mints on the same -// terms; only a genuinely change-free re-add (identical type and metadata) -// mints nothing. +// metadata or thread is the same kind of durable-state change and mints on the +// same terms; only a genuinely change-free re-add (identical type and metadata, +// no new thread) mints nothing. func AddDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, actor string, opts AddDependencyOpts) (bool, error) { return addDependencyInTx(ctx, tx, dep, actor, opts, nil, true) } @@ -295,38 +309,46 @@ func addDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a // target expression defensively so stale/reclassified rows in another typed // target column cannot bypass the idempotency/conflict check. var existingType string - var existingMetadataNS sql.NullString + var existingMetadataNS, existingThreadNS sql.NullString //nolint:gosec // G201: writeTable from WispTableRouting; depTargetEquals has no user input. - err := tx.QueryRowContext(ctx, fmt.Sprintf(`SELECT type, metadata FROM %s WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), - dep.IssueID, dep.DependsOnID).Scan(&existingType, &existingMetadataNS) + err := tx.QueryRowContext(ctx, fmt.Sprintf(`SELECT type, metadata, thread_id FROM %s WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), + dep.IssueID, dep.DependsOnID).Scan(&existingType, &existingMetadataNS, &existingThreadNS) if err == nil { existingMetadata := existingMetadataNS.String if !existingMetadataNS.Valid { existingMetadata = "{}" } if existingType == string(dep.Type) { - if DependencyMetadataEqual(existingMetadata, metadata) { - // Same type, same metadata: a change-free write. Nothing is - // written and nothing is journaled (#5898 R3). + // A re-add naming a thread the row does not carry moves it. One + // naming none keeps the stored thread rather than clearing it, so + // the callers that never carry a thread cannot erase one. + thread, threadChanged := ReAddedDependencyThread(existingThreadNS, dep.ThreadID) + if !threadChanged && DependencyMetadataEqual(existingMetadata, metadata) { + // Same type, same metadata, same thread: a change-free write. + // Nothing is written and nothing is journaled (#5898 R3). return false, nil } - // Same type, different metadata — idempotent; update metadata. No - // event is written, so the caller must not stage the events table - // for this re-add. + // Same type, different metadata or thread — idempotent; update the + // row in place. No event is written, so the caller must not stage + // the events table for this re-add. //nolint:gosec // G201: writeTable from WispTableRouting; depTargetEquals has no user input. - if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET metadata = ? WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), - metadata, dep.IssueID, dep.DependsOnID); err != nil { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET metadata = ?, thread_id = ? WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), + metadata, thread, dep.IssueID, dep.DependsOnID); err != nil { return false, fmt.Errorf("failed to update dependency metadata: %w", err) } - // A same-type add refreshes edge metadata. It is an observable graph - // mutation, so emit the complete replacement edge for replay even - // though no audit event is written. + // A same-type add refreshes the edge's metadata or thread. It is an + // observable graph mutation, so journal the replacement edge for + // replay even though no audit event is written. EventDep has no + // thread member, so a thread-only change journals the same payload + // as the original add; the source's version snapshot is what + // records the new thread. if err := RecordDepEventInTx(ctx, tx, EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor); err != nil { return false, err } - // The metadata genuinely changed, so — unlike the change-free - // branch above — this re-add is a real durable-state mutation of - // the source issue and mints on the same terms as a new edge + // The metadata or thread genuinely changed, so — unlike the + // change-free branch above — this re-add is a real durable-state + // mutation of the source issue (its version snapshot carries each + // edge's thread_id) and mints on the same terms as a new edge // (#5898 leg 2: "a same-type re-add whose metadata actually // changed mints EXACTLY ONE version carrying the new state"). return false, mintDependencyVersion(ctx, tx, dep.IssueID, actor, mintVersion) @@ -421,11 +443,11 @@ func addDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a } // mintDependencyVersion versions the referencing (source) issue of an edge that -// was actually inserted, deleted, or had its metadata refreshed by a same-type -// re-add: the version-history seam for every dependency write path, reached -// from addDependencyInTx and removeDependencyInTx only past a real row write — -// never from a genuinely change-free re-add or the absent-edge return, which -// mint nothing. mint false defers to a caller that mints once for a +// was actually inserted, deleted, or had its metadata or thread refreshed by a +// same-type re-add: the version-history seam for every dependency write path, +// reached from addDependencyInTx and removeDependencyInTx only past a real row +// write — never from a genuinely change-free re-add or the absent-edge return, +// which mint nothing. mint false defers to a caller that mints once for a // multi-edge mutation. A wisp source is excluded by the seam itself. func mintDependencyVersion(ctx context.Context, tx DBTX, issueID, actor string, mint bool) error { if !mint { diff --git a/internal/storage/issueops/dependencies_test.go b/internal/storage/issueops/dependencies_test.go index 635f81cb2..af2cfb5cd 100644 --- a/internal/storage/issueops/dependencies_test.go +++ b/internal/storage/issueops/dependencies_test.go @@ -2,6 +2,7 @@ package issueops import ( "context" + "database/sql" "errors" "reflect" "regexp" @@ -331,6 +332,45 @@ func TestDependencyMetadataEqualComparesValuesNotBytes(t *testing.T) { } } +// TestReAddedDependencyThreadKeepsTheStoredThreadUnlessANewOneIsNamed pins the +// thread half of the same-type re-add gate. A re-add naming a thread the row +// does not carry is a change that must reach the row; a re-add naming none +// says nothing about threads, so it must leave the stored one — NULL, empty +// or named — exactly as it is, or every plain `bd dep add` of a threaded edge +// would erase its thread. +func TestReAddedDependencyThreadKeepsTheStoredThreadUnlessANewOneIsNamed(t *testing.T) { + t.Parallel() + + null := sql.NullString{} + empty := sql.NullString{Valid: true} + named := sql.NullString{String: "thread-1", Valid: true} + for _, tc := range []struct { + name string + stored sql.NullString + requested string + want sql.NullString + wantChanged bool + }{ + {name: "NoneNamedKeepsNull", stored: null, requested: "", want: null}, + {name: "NoneNamedKeepsEmpty", stored: empty, requested: "", want: empty}, + {name: "NoneNamedKeepsTheStoredThread", stored: named, requested: "", want: named}, + {name: "SameThreadIsNoChange", stored: named, requested: "thread-1", want: named}, + {name: "NewThreadOverNull", stored: null, requested: "thread-1", want: named, wantChanged: true}, + {name: "NewThreadOverEmpty", stored: empty, requested: "thread-1", want: named, wantChanged: true}, + {name: "DifferentThreadReplacesIt", stored: named, requested: "thread-2", + want: sql.NullString{String: "thread-2", Valid: true}, wantChanged: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, changed := ReAddedDependencyThread(tc.stored, tc.requested) + if got != tc.want || changed != tc.wantChanged { + t.Errorf("ReAddedDependencyThread(%+v, %q) = (%+v, %v), want (%+v, %v)", + tc.stored, tc.requested, got, changed, tc.want, tc.wantChanged) + } + }) + } +} + // TestAddDependencyInTxReadsNullStoredMetadataAsEmptyObject pins the widened // SELECT's handling of a SQL-NULL metadata column. dependencies.metadata is // nullable (`JSON DEFAULT (JSON_OBJECT())`, no NOT NULL), so a row written out @@ -341,6 +381,11 @@ func TestDependencyMetadataEqualComparesValuesNotBytes(t *testing.T) { // re-add of that edge into an error. NULL is the absent-metadata state, so it // reads as `{}` and the change-free re-add of a metadata-free edge stays the // no-op it was. +// +// thread_id is nullable the same way (a VARCHAR with no NOT NULL), and the gate +// reads it too, so the row here holds NULL in both columns: a re-add naming no +// thread keeps whatever thread the row stores, NULL included, so this re-add +// is still change-free. func TestAddDependencyInTxReadsNullStoredMetadataAsEmptyObject(t *testing.T) { t.Parallel() @@ -349,9 +394,9 @@ func TestAddDependencyInTxReadsNullStoredMetadataAsEmptyObject(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta("SELECT issue_type FROM issues WHERE id = ?")). WithArgs("dep-a"). WillReturnRows(sqlmock.NewRows([]string{"issue_type"}).AddRow("task")) - mock.ExpectQuery(regexp.QuoteMeta("SELECT type, metadata FROM dependencies")). + mock.ExpectQuery(regexp.QuoteMeta("SELECT type, metadata, thread_id FROM dependencies")). WithArgs("dep-a", "dep-b"). - WillReturnRows(sqlmock.NewRows([]string{"type", "metadata"}).AddRow(string(types.DepRelated), nil)) + WillReturnRows(sqlmock.NewRows([]string{"type", "metadata", "thread_id"}).AddRow(string(types.DepRelated), nil, nil)) kind := DepTargetIssue dep := &types.Dependency{IssueID: "dep-a", DependsOnID: "dep-b", Type: types.DepRelated} diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index d49656b44..6ae8f3899 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -118,6 +118,15 @@ func TestBatchApplyContract(t *testing.T) { t.Run("StampsSpawnerIDOnlyWhenNamed", func(t *testing.T) { conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, ctx, fixture) }) + t.Run("CarriesThreadIDOntoTheStoredEdge", func(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoTheStoredEdge(t, ctx, fixture) + }) + t.Run("CarriesThreadIDOntoAnExistingEdge", func(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoAnExistingEdge(t, ctx, fixture) + }) + t.Run("RefusesAThreadIDLongerThanItsColumn", func(t *testing.T) { + conformance.RunBatchApplyRefusesAThreadIDLongerThanItsColumn(t, ctx, fixture) + }) t.Run("SplicesAForwardMetadataRef", func(t *testing.T) { conformance.RunBatchApplySplicesAForwardMetadataRef(t, ctx, fixture) }) diff --git a/internal/storage/uow/dependency_editor_contract_test.go b/internal/storage/uow/dependency_editor_contract_test.go index 02b91a878..855ec2521 100644 --- a/internal/storage/uow/dependency_editor_contract_test.go +++ b/internal/storage/uow/dependency_editor_contract_test.go @@ -32,6 +32,7 @@ func TestUOWDependencyEditorContract(t *testing.T) { {name: "SameTypeReAddIsIdempotent", run: conformance.RunDependencyEditorSameTypeReAddIsIdempotent}, {name: "SameTypeReAddWithChangedMetadataMintsOneVersion", run: conformance.RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion}, {name: "SameTypeReAddWithIdenticalMetadataIsANoOp", run: conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp}, + {name: "SameTypeReAddWithChangedThreadMintsOneVersion", run: conformance.RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion}, {name: "RepeatsWithinOneRequestCollapse", run: conformance.RunDependencyEditorRepeatsWithinOneRequestCollapse}, {name: "AttributesItsEventsToTheActor", run: conformance.RunDependencyEditorAttributesItsEventsToTheActor}, {name: "RetypeRefusalLeavesTheOriginalEdge", run: conformance.RunDependencyEditorRetypeRefusalLeavesTheOriginalEdge}, diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index 39889926e..ac84c74df 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -220,9 +220,13 @@ type DepAddItem struct { // not express. One spelling, and it is this one. Metadata string // HasSpawner marks a DepWaitsFor edge whose caller explicitly named a - // spawner (bd create --graph's edges[].spawner_key/spawner_id, which the - // CLI's own plan validator already forces to equal this edge's own - // Target). Only then does the role stamp Metadata's spawner_id from the + // spawner: a graph plan's edges[].spawner_key/spawner_id, which bd create + // --graph's plan validator forces to equal the edge's own Target. It has + // two producers: bd create --graph itself, which sets it from those + // members on the embedded and proxied legs alike (cmd/bd's + // buildGraphApplyBatchRequest), and an embedder lowering such a plan onto + // this role (gc's ApplyGraphPlan mapping, on BatchApplier below). Only + // when it is set does the role stamp Metadata's spawner_id from the // resolved Target, once every id in the batch exists (see // StampWaitsForSpawnerID) — a plan-local spawner key cannot be resolved // any earlier than that. @@ -236,9 +240,15 @@ type DepAddItem struct { // Type. HasSpawner bool // ThreadID carries conversation-threading metadata (e.g. a replies-to - // edge's originating thread) onto the created dependency row. Empty means - // the edge carries none; it is a plain column, not part of Metadata's - // type-specific blob. + // edge's originating thread) onto the dependency row. Empty means the edge + // carries none; it is a plain column, not part of Metadata's type-specific + // blob, and one longer than the column holds is ErrValidation. + // + // A RE-ADD OF AN EDGE THAT ALREADY EXISTS WITH THE SAME TYPE CARRIES IT + // TOO. A thread the stored row does not already carry replaces it, and + // versions the source on the same terms as a Metadata change; an empty + // ThreadID keeps the stored thread rather than clearing it, so a re-apply + // that names none cannot erase one. ThreadID string } @@ -558,6 +568,29 @@ func HasExtendedRetryBudget(ctx context.Context) bool { // directly. `bd create --file` and `bd dep add --file` remain the file-shaped // front doors for the two homogeneous cases they already serve. // +// TWO EMBEDDER OPERATIONS MAP ONTO THIS ROLE RATHER THAN EARNING THEIR OWN +// (DESIGN §3.8, "Guarded writes: NO NEW ROLES"). Neither needs a new role or a +// new wire member; both are a SHAPE of ApplyBatchRequest, pinned by an +// http-leg conformance case so the shape does not drift under either side +// independently: +// +// - CloseWithMetadataIfMatch is an update item carrying a Metadata patch and +// ExpectedVersion, followed by a close item on the SAME target, in ONE +// request: []ApplyItem{{Kind: ItemUpdate, Update: &UpdateItem{Target: t, +// Patch: IssuePatch{Metadata: ...}, ExpectedVersion: &v}}, {Kind: ItemClose, +// Close: &CloseItem{Target: t}}}. It is atomic because the whole request is: +// a stale ExpectedVersion refuses the update, which refuses the request, and +// the close never runs — zero rows change. CloseItem itself carries no +// metadata member; this composition is the only way to land a metadata +// write and a close as one guarded act. +// - ApplyGraphPlan is a create item per node (Key, the issue's Labels and +// Metadata, Ephemeral, NoHistory, and MetadataRefs for a value only the +// plan's own later ids can supply), dep_add items for every edge the plan +// names — parent-child included, since a parent is a dep_add item of type +// parent-child like any other edge — and trailing update items for an +// AssignAfterCreate step, which patches a just-created row's Assignee by +// its backward Key reference. One request, declaration order, one outcome. +// // Implementations never mutate caller-owned request values, snapshot the // request at method entry, and apply validation and normalization only to // attempt-local clones. Deterministic request-validation failures match From d90823480a6554f3b6969fe424c54f885f659e23 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 05:13:07 +0000 Subject: [PATCH 007/151] ci(bazel): split //scripts:scripts_test by what it reads; partition //:repo_files (#7414) * ci(bazel): split //scripts:scripts_test by what it reads; partition //:repo_files //scripts:scripts_test declared //:repo_files (the whole checkout), so every PR re-ran it, and it took ~181s on rbe-west. Most of that (60-79s) was TestBazelGatedLanesNeverRetryFlakyTests line-regexping the --run_under wrapper's pinned dolt binary (tools/bazel/hermetic_bin/dolt, in every test's runfiles); it now skips binary files, as git grep -I does. tools/bazel/go_srcs.py's per-package repo_files block now partitions each package's files into repo_go_srcs, repo_go_test_srcs, repo_doc_files and repo_other_files, aggregated at the root; //:repo_files is their union (same 4679 files). The //scripts tests are split into go_test targets by what they read (traced per test with strace over a runfiles-shaped tree): scripts_test CI/Bazel policy: //:repo_other_files (+1 Go file) shell_scripts_test script fixtures: //scripts files, a few root files go_test_sources_test _test.go scans: //:repo_go_test_srcs + shard files go_sources_test check-testing-short.sh: every .go file tracked_files_test every tracked file: //:repo_files Shared helpers move to helpers-only files; TestScriptsTestTargetsPartition- TheTests fails on a test file in no target or in two. //test/docsync and the //scripts/repochecks slices declare partitions or single files instead of //:repo_files where they can. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): skip only the non-checkout hermetic_bin dir in the flaky-retry walk Skipping every binary file also stopped scanning tracked binaries (images, .binpb fixtures). Skip just tools/bazel/hermetic_bin, the --run_under wrapper's runfiles directory, which is not in the checkout, so the walk's coverage of tracked files is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- BUILD.bazel | 1061 ++++++++++++++--- MODULE.bazel | 2 +- backend/BUILD.bazel | 43 +- backend/conformance/BUILD.bazel | 43 +- beadserrors/BUILD.bazel | 43 +- cmd/bd/BUILD.bazel | 57 +- cmd/bd/doctor/BUILD.bazel | 43 +- cmd/bd/doctor/fix/BUILD.bazel | 43 +- cmd/bd/protocol/BUILD.bazel | 43 +- cmd/bd/setup/BUILD.bazel | 43 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 17 +- format/BUILD.bazel | 43 +- internal/ado/BUILD.bazel | 43 +- internal/atomicfile/BUILD.bazel | 43 +- internal/audit/BUILD.bazel | 43 +- internal/beads/BUILD.bazel | 43 +- internal/ceiling/BUILD.bazel | 43 +- internal/compact/BUILD.bazel | 43 +- internal/config/BUILD.bazel | 43 +- internal/configfile/BUILD.bazel | 43 +- internal/creds/BUILD.bazel | 43 +- internal/debug/BUILD.bazel | 43 +- internal/doltremote/BUILD.bazel | 43 +- internal/doltserver/BUILD.bazel | 43 +- internal/doltversion/BUILD.bazel | 43 +- internal/eventsjournal/BUILD.bazel | 43 +- internal/execenv/BUILD.bazel | 43 +- internal/fdhygiene/BUILD.bazel | 43 +- internal/formula/BUILD.bazel | 43 +- internal/formula/cmd/schemagen/BUILD.bazel | 43 +- internal/formula/schemagen/BUILD.bazel | 43 +- internal/git/BUILD.bazel | 43 +- internal/gitenv/BUILD.bazel | 43 +- internal/githooksenv/BUILD.bazel | 43 +- internal/github/BUILD.bazel | 43 +- internal/gitignore/BUILD.bazel | 43 +- internal/gitlab/BUILD.bazel | 43 +- internal/gittraceenv/BUILD.bazel | 43 +- internal/hooks/BUILD.bazel | 43 +- internal/httpapi/BUILD.bazel | 43 +- internal/httpapi/apigen/BUILD.bazel | 43 +- internal/httpapi/spec/BUILD.bazel | 43 +- internal/httpapi/wireshape/BUILD.bazel | 43 +- .../wireshape/cmd/gendigest/BUILD.bazel | 43 +- internal/httpclient/BUILD.bazel | 43 +- internal/httpclient/encode/BUILD.bazel | 43 +- internal/httpclient/wire/BUILD.bazel | 43 +- internal/idgen/BUILD.bazel | 43 +- internal/jira/BUILD.bazel | 43 +- internal/linear/BUILD.bazel | 43 +- internal/lockfile/BUILD.bazel | 43 +- internal/memoryapi/BUILD.bazel | 43 +- internal/metrics/BUILD.bazel | 43 +- internal/migration/BUILD.bazel | 43 +- internal/migration/legacysqlite/BUILD.bazel | 43 +- internal/molecules/BUILD.bazel | 43 +- internal/notion/BUILD.bazel | 43 +- internal/procid/BUILD.bazel | 43 +- internal/query/BUILD.bazel | 43 +- internal/recipes/BUILD.bazel | 43 +- internal/remotecache/BUILD.bazel | 43 +- internal/routing/BUILD.bazel | 43 +- internal/storage/BUILD.bazel | 43 +- internal/storage/backendnames/BUILD.bazel | 43 +- internal/storage/backends/BUILD.bazel | 43 +- internal/storage/batchbench/BUILD.bazel | 43 +- internal/storage/batchfixtures/BUILD.bazel | 43 +- internal/storage/contextinfo/BUILD.bazel | 43 +- internal/storage/createbatchequiv/BUILD.bazel | 43 +- internal/storage/dberrors/BUILD.bazel | 43 +- internal/storage/dbproxy/identity/BUILD.bazel | 43 +- internal/storage/dbproxy/pidfile/BUILD.bazel | 43 +- internal/storage/dbproxy/proxy/BUILD.bazel | 43 +- internal/storage/dbproxy/server/BUILD.bazel | 43 +- internal/storage/dbproxy/util/BUILD.bazel | 43 +- internal/storage/depid/BUILD.bazel | 43 +- internal/storage/dolt/BUILD.bazel | 43 +- internal/storage/doltutil/BUILD.bazel | 43 +- internal/storage/domain/BUILD.bazel | 43 +- internal/storage/domain/db/BUILD.bazel | 43 +- internal/storage/domain/fs/BUILD.bazel | 43 +- internal/storage/domain/git/BUILD.bazel | 43 +- internal/storage/embeddeddolt/BUILD.bazel | 43 +- internal/storage/embeddeddolt/cmd/BUILD.bazel | 43 +- internal/storage/externaldeps/BUILD.bazel | 43 +- internal/storage/fs/BUILD.bazel | 43 +- internal/storage/git/BUILD.bazel | 43 +- internal/storage/issueops/BUILD.bazel | 43 +- internal/storage/journalscan/BUILD.bazel | 43 +- internal/storage/kvkeys/BUILD.bazel | 43 +- internal/storage/memoryops/BUILD.bazel | 43 +- internal/storage/rowid/BUILD.bazel | 43 +- internal/storage/schema/BUILD.bazel | 43 +- internal/storage/sqlbuild/BUILD.bazel | 43 +- internal/storage/sqlclass/BUILD.bazel | 43 +- internal/storage/sqlcount/BUILD.bazel | 43 +- internal/storage/storagecontract/BUILD.bazel | 43 +- internal/storage/unsupportedgen/BUILD.bazel | 43 +- internal/storage/uow/BUILD.bazel | 43 +- .../storage/versioncontrolops/BUILD.bazel | 43 +- internal/telemetry/BUILD.bazel | 43 +- internal/templates/BUILD.bazel | 43 +- internal/templates/agents/BUILD.bazel | 43 +- internal/testutil/BUILD.bazel | 53 +- internal/testutil/bazeltest/BUILD.bazel | 43 +- internal/testutil/credentialcmd/BUILD.bazel | 43 +- internal/testutil/fixtures/BUILD.bazel | 43 +- internal/testutil/integration/BUILD.bazel | 43 +- internal/timeparsing/BUILD.bazel | 43 +- internal/tracker/BUILD.bazel | 43 +- internal/tracker/conformance/BUILD.bazel | 43 +- internal/types/BUILD.bazel | 43 +- internal/ui/BUILD.bazel | 43 +- internal/uimd/BUILD.bazel | 43 +- internal/utils/BUILD.bazel | 43 +- internal/validation/BUILD.bazel | 43 +- internal/versioncheck/BUILD.bazel | 43 +- internal/workapi/BUILD.bazel | 43 +- internal/workapi/storecounter/BUILD.bazel | 43 +- internal/workapi/storequerier/BUILD.bazel | 43 +- internal/workapi/storereader/BUILD.bazel | 43 +- .../workapi/storereadycounter/BUILD.bazel | 43 +- internal/workapi/storestats/BUILD.bazel | 43 +- .../storeversionreconciler/BUILD.bazel | 43 +- .../workapi/storeworkspaceconfig/BUILD.bazel | 43 +- internal/workspacegate/BUILD.bazel | 43 +- internal/worktreeremove/BUILD.bazel | 43 +- issueops/BUILD.bazel | 43 +- journalops/BUILD.bazel | 43 +- memoryops/BUILD.bazel | 43 +- platforms/BUILD.bazel | 43 +- plugins/beads/BUILD.bazel | 43 +- schema/BUILD.bazel | 43 +- scripts/BUILD.bazel | 233 +++- scripts/README.md | 2 +- scripts/bazel_autofix_push_test.go | 11 - scripts/bazel_policy_test.go | 392 +----- scripts/bazel_skip_allowlist_test.go | 20 +- scripts/bench-ready-indexes/BUILD.bazel | 43 +- scripts/check-versions/BUILD.bazel | 43 +- scripts/check_testing_short_test.go | 20 - scripts/check_winget_portable_alias_test.go | 2 +- scripts/checkworkflowtags/BUILD.bazel | 43 +- scripts/ci_workflow_test.go | 74 +- scripts/embedded_shard_count_test.go | 72 +- scripts/gitattributespolicy/BUILD.bazel | 43 +- scripts/host_tools_test.go | 15 +- scripts/internal/testbash/BUILD.bazel | 43 +- scripts/migration-test/catalog/BUILD.bazel | 43 +- scripts/pr-lint/BUILD.bazel | 43 +- scripts/pr_lanes_bazel_coverage_test.go | 4 +- scripts/pr_risk_bazel_coverage_test.go | 202 +--- scripts/prlintmake/BUILD.bazel | 43 +- scripts/pull_dolt_image_test.go | 30 - scripts/repo_helpers_test.go | 151 ++- scripts/repochecks/BUILD.bazel | 85 +- scripts/repro-dolt-hang/BUILD.bazel | 43 +- scripts/repro-dolt-prod-timeouts/BUILD.bazel | 43 +- scripts/scripts_targets_test.go | 105 ++ scripts/shard_manifest_test.go | 251 ++++ scripts/testing_short_tree_test.go | 33 + scripts/tracked_files_test.go | 295 +++++ scripts/workflowtags/BUILD.bazel | 43 +- test/conformance/BUILD.bazel | 43 +- test/docsync/BUILD.bazel | 70 +- test/docsync/agentfiles_test.go | 9 +- test/docsync/docsync_test.go | 13 +- test/testmainconvention/BUILD.bazel | 43 +- tests/hook_timeout_backends/BUILD.bazel | 43 +- tests/migration/BUILD.bazel | 43 +- tests/regression/BUILD.bazel | 43 +- tests/upgrade_smoke/BUILD.bazel | 43 +- third_party/cc_toolchain/BUILD.bazel | 43 +- third_party/patches/BUILD.bazel | 43 +- tools/bazel/BUILD.bazel | 53 +- tools/bazel/go_srcs.py | 117 +- tools/docsmint/BUILD.bazel | 43 +- tools/nogo/BUILD.bazel | 49 +- tools/nogo/analyzers/depguard/BUILD.bazel | 43 +- tools/nogo/analyzers/errcheck/BUILD.bazel | 43 +- tools/nogo/analyzers/forbidigo/BUILD.bazel | 43 +- tools/nogo/analyzers/gosec/BUILD.bazel | 43 +- tools/nogo/analyzers/misspell/BUILD.bazel | 43 +- tools/nogo/analyzers/sloglint/BUILD.bazel | 43 +- tools/nogo/analyzers/unconvert/BUILD.bazel | 43 +- tools/nogo/analyzers/unparam/BUILD.bazel | 43 +- tools/nogo/internal/analyzertest/BUILD.bazel | 43 +- tools/nogo/internal/golangci/BUILD.bazel | 43 +- tools/nogo/internal/position/BUILD.bazel | 43 +- 189 files changed, 8985 insertions(+), 1350 deletions(-) create mode 100644 scripts/scripts_targets_test.go create mode 100644 scripts/shard_manifest_test.go create mode 100644 scripts/testing_short_tree_test.go create mode 100644 scripts/tracked_files_test.go diff --git a/BUILD.bazel b/BUILD.bazel index 3699152e2..69157bf54 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -22,6 +22,7 @@ gazelle(name = "gazelle") exports_files( [".buildflags"], visibility = [ + "//scripts:__pkg__", "//scripts/prlintmake:__pkg__", "//tests/upgrade_smoke:__pkg__", ], @@ -33,9 +34,11 @@ exports_files([ ]) # The golangci-lint configuration nogo applies (//tools/nogo), and its test. +# //test/docsync checks the agent instructions' links to it. exports_files( [".golangci.yml"], visibility = [ + "//test/docsync:__pkg__", "//tools/nogo:__pkg__", "//tools/nogo/internal/golangci:__pkg__", ], @@ -49,7 +52,8 @@ exports_files( ) # //tests/hook_timeout_backends: the tracked git hooks whose managed section -# it runs against real timeout implementations. +# it runs against real timeout implementations. //scripts:shell_scripts_test +# runs pre-push's suite dispatch. exports_files( [ ".githooks/post-checkout", @@ -58,7 +62,49 @@ exports_files( ".githooks/pre-push", ".githooks/prepare-commit-msg", ], - visibility = ["//tests/hook_timeout_backends:__pkg__"], + visibility = [ + "//scripts:__pkg__", + "//tests/hook_timeout_backends:__pkg__", + ], +) + +# //scripts:shell_scripts_test: scripts/test.sh's skip list. +exports_files( + [".test-skip"], + visibility = ["//scripts:__pkg__"], +) + +# The nested Go modules' go.mod files, which mark where the root module ends: +# //scripts:go_test_sources_test's scan of every _test.go file skips their +# trees, as `go test ./...` does. (go.mod is in no Go partition below: it is +# in //:repo_other_files.) +filegroup( + name = "nested_go_modules", + srcs = [ + "examples/bd-example-extension-go/go.mod", + "examples/library-usage/go.mod", + "//tools/nogo:go.mod", + ], + visibility = ["//scripts:__pkg__"], +) + +# //scripts/repochecks:winget_portable_alias_test: the winget manifests. +filegroup( + name = "winget_manifests", + srcs = glob(["winget/**"]), + visibility = ["//scripts/repochecks:__pkg__"], +) + +# //test/docsync: the repository files outside its Markdown and Go-test +# partitions that the docs and agent instructions link to by path. A new link +# to a file not listed here fails docsync_test under Bazel; add the file. +exports_files( + [ + ".github/workflows/bazel.yml", + "beads.go", + "docs/docs.json", + ], + visibility = ["//test/docsync:__pkg__"], ) # //cmd/bd:bd_test: tracked git hooks (generator parity) and the formula @@ -83,7 +129,8 @@ exports_files( # //internal/storage/embeddeddolt:embeddeddolt_embedded_test, # //internal/storage/dolt:dolt_server_full_test) run them through # tools/bazel/go_test_manifest_shard.sh, so each Bazel shard runs exactly the -# tests of the CI job with the same shard number. +# tests of the CI job with the same shard number. //scripts:go_test_sources_test +# checks the manifests and scripts against the tests they shard. exports_files( [ ".github/scripts/embedded-cmd-test-shards.txt", @@ -99,6 +146,7 @@ exports_files( "//cmd/bd:__pkg__", "//internal/storage/dolt:__pkg__", "//internal/storage/embeddeddolt:__pkg__", + "//scripts:__pkg__", ], ) @@ -199,13 +247,682 @@ exports_files( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = [ + "**/node_modules/**", + ".agents/**", + ".amp/**", + ".augment/**", + ".bazelrc.local", + ".claude/*.lock", + ".claude/*.log", + ".claude/settings.local.json", + ".codex/**", + ".cursor/**", + ".direnv/**", + ".envrc", + ".git/**", + ".idea/**", + ".logs/**", + ".vscode/**", + "bazel-*/**", + "bd", + "bd-fixed", + "bd-original", + "bd-test", + "bd_test", + "beads", + "go.work", + "go.work.sum", + "history/**", + "mcp_agent_mail/**", + "npm-package/bin/*.tar.gz", + "npm-package/bin/*.zip", + "npm-package/bin/CHANGELOG.md", + "npm-package/bin/LICENSE", + "npm-package/bin/README.md", + "npm-package/bin/bd", + "npm-package/package-lock.json", + "output", + "result", + "state.json", + "user.bazelrc", + ], + ) + [ + "//backend:repo_doc_files", + "//backend/conformance:repo_doc_files", + "//beadserrors:repo_doc_files", + "//cmd/bd:repo_doc_files", + "//cmd/bd/doctor:repo_doc_files", + "//cmd/bd/doctor/fix:repo_doc_files", + "//cmd/bd/protocol:repo_doc_files", + "//cmd/bd/setup:repo_doc_files", + "//format:repo_doc_files", + "//internal/ado:repo_doc_files", + "//internal/atomicfile:repo_doc_files", + "//internal/audit:repo_doc_files", + "//internal/beads:repo_doc_files", + "//internal/ceiling:repo_doc_files", + "//internal/compact:repo_doc_files", + "//internal/config:repo_doc_files", + "//internal/configfile:repo_doc_files", + "//internal/creds:repo_doc_files", + "//internal/debug:repo_doc_files", + "//internal/doltremote:repo_doc_files", + "//internal/doltserver:repo_doc_files", + "//internal/doltversion:repo_doc_files", + "//internal/eventsjournal:repo_doc_files", + "//internal/execenv:repo_doc_files", + "//internal/fdhygiene:repo_doc_files", + "//internal/formula:repo_doc_files", + "//internal/formula/cmd/schemagen:repo_doc_files", + "//internal/formula/schemagen:repo_doc_files", + "//internal/git:repo_doc_files", + "//internal/gitenv:repo_doc_files", + "//internal/githooksenv:repo_doc_files", + "//internal/github:repo_doc_files", + "//internal/gitignore:repo_doc_files", + "//internal/gitlab:repo_doc_files", + "//internal/gittraceenv:repo_doc_files", + "//internal/hooks:repo_doc_files", + "//internal/httpapi:repo_doc_files", + "//internal/httpapi/apigen:repo_doc_files", + "//internal/httpapi/spec:repo_doc_files", + "//internal/httpapi/wireshape:repo_doc_files", + "//internal/httpapi/wireshape/cmd/gendigest:repo_doc_files", + "//internal/httpclient:repo_doc_files", + "//internal/httpclient/encode:repo_doc_files", + "//internal/httpclient/wire:repo_doc_files", + "//internal/idgen:repo_doc_files", + "//internal/jira:repo_doc_files", + "//internal/linear:repo_doc_files", + "//internal/lockfile:repo_doc_files", + "//internal/memoryapi:repo_doc_files", + "//internal/metrics:repo_doc_files", + "//internal/migration:repo_doc_files", + "//internal/migration/legacysqlite:repo_doc_files", + "//internal/molecules:repo_doc_files", + "//internal/notion:repo_doc_files", + "//internal/procid:repo_doc_files", + "//internal/query:repo_doc_files", + "//internal/recipes:repo_doc_files", + "//internal/remotecache:repo_doc_files", + "//internal/routing:repo_doc_files", + "//internal/storage:repo_doc_files", + "//internal/storage/backendnames:repo_doc_files", + "//internal/storage/backends:repo_doc_files", + "//internal/storage/batchbench:repo_doc_files", + "//internal/storage/batchfixtures:repo_doc_files", + "//internal/storage/contextinfo:repo_doc_files", + "//internal/storage/createbatchequiv:repo_doc_files", + "//internal/storage/dberrors:repo_doc_files", + "//internal/storage/dbproxy/identity:repo_doc_files", + "//internal/storage/dbproxy/pidfile:repo_doc_files", + "//internal/storage/dbproxy/proxy:repo_doc_files", + "//internal/storage/dbproxy/server:repo_doc_files", + "//internal/storage/dbproxy/util:repo_doc_files", + "//internal/storage/depid:repo_doc_files", + "//internal/storage/dolt:repo_doc_files", + "//internal/storage/doltutil:repo_doc_files", + "//internal/storage/domain:repo_doc_files", + "//internal/storage/domain/db:repo_doc_files", + "//internal/storage/domain/fs:repo_doc_files", + "//internal/storage/domain/git:repo_doc_files", + "//internal/storage/embeddeddolt:repo_doc_files", + "//internal/storage/embeddeddolt/cmd:repo_doc_files", + "//internal/storage/externaldeps:repo_doc_files", + "//internal/storage/fs:repo_doc_files", + "//internal/storage/git:repo_doc_files", + "//internal/storage/issueops:repo_doc_files", + "//internal/storage/journalscan:repo_doc_files", + "//internal/storage/kvkeys:repo_doc_files", + "//internal/storage/memoryops:repo_doc_files", + "//internal/storage/rowid:repo_doc_files", + "//internal/storage/schema:repo_doc_files", + "//internal/storage/sqlbuild:repo_doc_files", + "//internal/storage/sqlclass:repo_doc_files", + "//internal/storage/sqlcount:repo_doc_files", + "//internal/storage/storagecontract:repo_doc_files", + "//internal/storage/unsupportedgen:repo_doc_files", + "//internal/storage/uow:repo_doc_files", + "//internal/storage/versioncontrolops:repo_doc_files", + "//internal/telemetry:repo_doc_files", + "//internal/templates:repo_doc_files", + "//internal/templates/agents:repo_doc_files", + "//internal/testutil:repo_doc_files", + "//internal/testutil/bazeltest:repo_doc_files", + "//internal/testutil/credentialcmd:repo_doc_files", + "//internal/testutil/fixtures:repo_doc_files", + "//internal/testutil/integration:repo_doc_files", + "//internal/timeparsing:repo_doc_files", + "//internal/tracker:repo_doc_files", + "//internal/tracker/conformance:repo_doc_files", + "//internal/types:repo_doc_files", + "//internal/ui:repo_doc_files", + "//internal/uimd:repo_doc_files", + "//internal/utils:repo_doc_files", + "//internal/validation:repo_doc_files", + "//internal/versioncheck:repo_doc_files", + "//internal/workapi:repo_doc_files", + "//internal/workapi/storecounter:repo_doc_files", + "//internal/workapi/storequerier:repo_doc_files", + "//internal/workapi/storereader:repo_doc_files", + "//internal/workapi/storereadycounter:repo_doc_files", + "//internal/workapi/storestats:repo_doc_files", + "//internal/workapi/storeversionreconciler:repo_doc_files", + "//internal/workapi/storeworkspaceconfig:repo_doc_files", + "//internal/workspacegate:repo_doc_files", + "//internal/worktreeremove:repo_doc_files", + "//issueops:repo_doc_files", + "//journalops:repo_doc_files", + "//memoryops:repo_doc_files", + "//platforms:repo_doc_files", + "//plugins/beads:repo_doc_files", + "//schema:repo_doc_files", + "//scripts:repo_doc_files", + "//scripts/bench-ready-indexes:repo_doc_files", + "//scripts/check-versions:repo_doc_files", + "//scripts/checkworkflowtags:repo_doc_files", + "//scripts/gitattributespolicy:repo_doc_files", + "//scripts/internal/testbash:repo_doc_files", + "//scripts/migration-test/catalog:repo_doc_files", + "//scripts/pr-lint:repo_doc_files", + "//scripts/prlintmake:repo_doc_files", + "//scripts/repochecks:repo_doc_files", + "//scripts/repro-dolt-hang:repo_doc_files", + "//scripts/repro-dolt-prod-timeouts:repo_doc_files", + "//scripts/workflowtags:repo_doc_files", + "//test/conformance:repo_doc_files", + "//test/docsync:repo_doc_files", + "//test/testmainconvention:repo_doc_files", + "//tests/hook_timeout_backends:repo_doc_files", + "//tests/migration:repo_doc_files", + "//tests/regression:repo_doc_files", + "//tests/upgrade_smoke:repo_doc_files", + "//third_party/cc_toolchain:repo_doc_files", + "//third_party/patches:repo_doc_files", + "//tools/bazel:repo_doc_files", + "//tools/docsmint:repo_doc_files", + "//tools/nogo:repo_doc_files", + "//tools/nogo/analyzers/depguard:repo_doc_files", + "//tools/nogo/analyzers/errcheck:repo_doc_files", + "//tools/nogo/analyzers/forbidigo:repo_doc_files", + "//tools/nogo/analyzers/gosec:repo_doc_files", + "//tools/nogo/analyzers/misspell:repo_doc_files", + "//tools/nogo/analyzers/sloglint:repo_doc_files", + "//tools/nogo/analyzers/unconvert:repo_doc_files", + "//tools/nogo/analyzers/unparam:repo_doc_files", + "//tools/nogo/internal/analyzertest:repo_doc_files", + "//tools/nogo/internal/golangci:repo_doc_files", + "//tools/nogo/internal/position:repo_doc_files", + ], + visibility = ["//:__subpackages__"], +) + filegroup( name = "repo_files", + srcs = [ + ":repo_doc_files", + ":repo_go_srcs", + ":repo_go_test_srcs", + ":repo_other_files", + ], + visibility = [ + "//scripts:__pkg__", + "//scripts/repochecks:__pkg__", + ], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ".agents/**", + ".amp/**", + ".augment/**", + ".bazelrc.local", + ".claude/*.lock", + ".claude/*.log", + ".claude/settings.local.json", + ".codex/**", + ".cursor/**", + ".direnv/**", + ".envrc", + ".git/**", + ".idea/**", + ".logs/**", + ".vscode/**", + "bazel-*/**", + "bd", + "bd-fixed", + "bd-original", + "bd-test", + "bd_test", + "beads", + "go.work", + "go.work.sum", + "history/**", + "mcp_agent_mail/**", + "npm-package/bin/*.tar.gz", + "npm-package/bin/*.zip", + "npm-package/bin/CHANGELOG.md", + "npm-package/bin/LICENSE", + "npm-package/bin/README.md", + "npm-package/bin/bd", + "npm-package/package-lock.json", + "output", + "result", + "state.json", + "user.bazelrc", + ], + ) + [ + "//backend:repo_go_srcs", + "//backend/conformance:repo_go_srcs", + "//beadserrors:repo_go_srcs", + "//cmd/bd:repo_go_srcs", + "//cmd/bd/doctor:repo_go_srcs", + "//cmd/bd/doctor/fix:repo_go_srcs", + "//cmd/bd/protocol:repo_go_srcs", + "//cmd/bd/setup:repo_go_srcs", + "//format:repo_go_srcs", + "//internal/ado:repo_go_srcs", + "//internal/atomicfile:repo_go_srcs", + "//internal/audit:repo_go_srcs", + "//internal/beads:repo_go_srcs", + "//internal/ceiling:repo_go_srcs", + "//internal/compact:repo_go_srcs", + "//internal/config:repo_go_srcs", + "//internal/configfile:repo_go_srcs", + "//internal/creds:repo_go_srcs", + "//internal/debug:repo_go_srcs", + "//internal/doltremote:repo_go_srcs", + "//internal/doltserver:repo_go_srcs", + "//internal/doltversion:repo_go_srcs", + "//internal/eventsjournal:repo_go_srcs", + "//internal/execenv:repo_go_srcs", + "//internal/fdhygiene:repo_go_srcs", + "//internal/formula:repo_go_srcs", + "//internal/formula/cmd/schemagen:repo_go_srcs", + "//internal/formula/schemagen:repo_go_srcs", + "//internal/git:repo_go_srcs", + "//internal/gitenv:repo_go_srcs", + "//internal/githooksenv:repo_go_srcs", + "//internal/github:repo_go_srcs", + "//internal/gitignore:repo_go_srcs", + "//internal/gitlab:repo_go_srcs", + "//internal/gittraceenv:repo_go_srcs", + "//internal/hooks:repo_go_srcs", + "//internal/httpapi:repo_go_srcs", + "//internal/httpapi/apigen:repo_go_srcs", + "//internal/httpapi/spec:repo_go_srcs", + "//internal/httpapi/wireshape:repo_go_srcs", + "//internal/httpapi/wireshape/cmd/gendigest:repo_go_srcs", + "//internal/httpclient:repo_go_srcs", + "//internal/httpclient/encode:repo_go_srcs", + "//internal/httpclient/wire:repo_go_srcs", + "//internal/idgen:repo_go_srcs", + "//internal/jira:repo_go_srcs", + "//internal/linear:repo_go_srcs", + "//internal/lockfile:repo_go_srcs", + "//internal/memoryapi:repo_go_srcs", + "//internal/metrics:repo_go_srcs", + "//internal/migration:repo_go_srcs", + "//internal/migration/legacysqlite:repo_go_srcs", + "//internal/molecules:repo_go_srcs", + "//internal/notion:repo_go_srcs", + "//internal/procid:repo_go_srcs", + "//internal/query:repo_go_srcs", + "//internal/recipes:repo_go_srcs", + "//internal/remotecache:repo_go_srcs", + "//internal/routing:repo_go_srcs", + "//internal/storage:repo_go_srcs", + "//internal/storage/backendnames:repo_go_srcs", + "//internal/storage/backends:repo_go_srcs", + "//internal/storage/batchbench:repo_go_srcs", + "//internal/storage/batchfixtures:repo_go_srcs", + "//internal/storage/contextinfo:repo_go_srcs", + "//internal/storage/createbatchequiv:repo_go_srcs", + "//internal/storage/dberrors:repo_go_srcs", + "//internal/storage/dbproxy/identity:repo_go_srcs", + "//internal/storage/dbproxy/pidfile:repo_go_srcs", + "//internal/storage/dbproxy/proxy:repo_go_srcs", + "//internal/storage/dbproxy/server:repo_go_srcs", + "//internal/storage/dbproxy/util:repo_go_srcs", + "//internal/storage/depid:repo_go_srcs", + "//internal/storage/dolt:repo_go_srcs", + "//internal/storage/doltutil:repo_go_srcs", + "//internal/storage/domain:repo_go_srcs", + "//internal/storage/domain/db:repo_go_srcs", + "//internal/storage/domain/fs:repo_go_srcs", + "//internal/storage/domain/git:repo_go_srcs", + "//internal/storage/embeddeddolt:repo_go_srcs", + "//internal/storage/embeddeddolt/cmd:repo_go_srcs", + "//internal/storage/externaldeps:repo_go_srcs", + "//internal/storage/fs:repo_go_srcs", + "//internal/storage/git:repo_go_srcs", + "//internal/storage/issueops:repo_go_srcs", + "//internal/storage/journalscan:repo_go_srcs", + "//internal/storage/kvkeys:repo_go_srcs", + "//internal/storage/memoryops:repo_go_srcs", + "//internal/storage/rowid:repo_go_srcs", + "//internal/storage/schema:repo_go_srcs", + "//internal/storage/sqlbuild:repo_go_srcs", + "//internal/storage/sqlclass:repo_go_srcs", + "//internal/storage/sqlcount:repo_go_srcs", + "//internal/storage/storagecontract:repo_go_srcs", + "//internal/storage/unsupportedgen:repo_go_srcs", + "//internal/storage/uow:repo_go_srcs", + "//internal/storage/versioncontrolops:repo_go_srcs", + "//internal/telemetry:repo_go_srcs", + "//internal/templates:repo_go_srcs", + "//internal/templates/agents:repo_go_srcs", + "//internal/testutil:repo_go_srcs", + "//internal/testutil/bazeltest:repo_go_srcs", + "//internal/testutil/credentialcmd:repo_go_srcs", + "//internal/testutil/fixtures:repo_go_srcs", + "//internal/testutil/integration:repo_go_srcs", + "//internal/timeparsing:repo_go_srcs", + "//internal/tracker:repo_go_srcs", + "//internal/tracker/conformance:repo_go_srcs", + "//internal/types:repo_go_srcs", + "//internal/ui:repo_go_srcs", + "//internal/uimd:repo_go_srcs", + "//internal/utils:repo_go_srcs", + "//internal/validation:repo_go_srcs", + "//internal/versioncheck:repo_go_srcs", + "//internal/workapi:repo_go_srcs", + "//internal/workapi/storecounter:repo_go_srcs", + "//internal/workapi/storequerier:repo_go_srcs", + "//internal/workapi/storereader:repo_go_srcs", + "//internal/workapi/storereadycounter:repo_go_srcs", + "//internal/workapi/storestats:repo_go_srcs", + "//internal/workapi/storeversionreconciler:repo_go_srcs", + "//internal/workapi/storeworkspaceconfig:repo_go_srcs", + "//internal/workspacegate:repo_go_srcs", + "//internal/worktreeremove:repo_go_srcs", + "//issueops:repo_go_srcs", + "//journalops:repo_go_srcs", + "//memoryops:repo_go_srcs", + "//platforms:repo_go_srcs", + "//plugins/beads:repo_go_srcs", + "//schema:repo_go_srcs", + "//scripts:repo_go_srcs", + "//scripts/bench-ready-indexes:repo_go_srcs", + "//scripts/check-versions:repo_go_srcs", + "//scripts/checkworkflowtags:repo_go_srcs", + "//scripts/gitattributespolicy:repo_go_srcs", + "//scripts/internal/testbash:repo_go_srcs", + "//scripts/migration-test/catalog:repo_go_srcs", + "//scripts/pr-lint:repo_go_srcs", + "//scripts/prlintmake:repo_go_srcs", + "//scripts/repochecks:repo_go_srcs", + "//scripts/repro-dolt-hang:repo_go_srcs", + "//scripts/repro-dolt-prod-timeouts:repo_go_srcs", + "//scripts/workflowtags:repo_go_srcs", + "//test/conformance:repo_go_srcs", + "//test/docsync:repo_go_srcs", + "//test/testmainconvention:repo_go_srcs", + "//tests/hook_timeout_backends:repo_go_srcs", + "//tests/migration:repo_go_srcs", + "//tests/regression:repo_go_srcs", + "//tests/upgrade_smoke:repo_go_srcs", + "//third_party/cc_toolchain:repo_go_srcs", + "//third_party/patches:repo_go_srcs", + "//tools/bazel:repo_go_srcs", + "//tools/docsmint:repo_go_srcs", + "//tools/nogo:repo_go_srcs", + "//tools/nogo/analyzers/depguard:repo_go_srcs", + "//tools/nogo/analyzers/errcheck:repo_go_srcs", + "//tools/nogo/analyzers/forbidigo:repo_go_srcs", + "//tools/nogo/analyzers/gosec:repo_go_srcs", + "//tools/nogo/analyzers/misspell:repo_go_srcs", + "//tools/nogo/analyzers/sloglint:repo_go_srcs", + "//tools/nogo/analyzers/unconvert:repo_go_srcs", + "//tools/nogo/analyzers/unparam:repo_go_srcs", + "//tools/nogo/internal/analyzertest:repo_go_srcs", + "//tools/nogo/internal/golangci:repo_go_srcs", + "//tools/nogo/internal/position:repo_go_srcs", + ], + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = [ + "**/node_modules/**", + ".agents/**", + ".amp/**", + ".augment/**", + ".bazelrc.local", + ".claude/*.lock", + ".claude/*.log", + ".claude/settings.local.json", + ".codex/**", + ".cursor/**", + ".direnv/**", + ".envrc", + ".git/**", + ".idea/**", + ".logs/**", + ".vscode/**", + "bazel-*/**", + "bd", + "bd-fixed", + "bd-original", + "bd-test", + "bd_test", + "beads", + "go.work", + "go.work.sum", + "history/**", + "mcp_agent_mail/**", + "npm-package/bin/*.tar.gz", + "npm-package/bin/*.zip", + "npm-package/bin/CHANGELOG.md", + "npm-package/bin/LICENSE", + "npm-package/bin/README.md", + "npm-package/bin/bd", + "npm-package/package-lock.json", + "output", + "result", + "state.json", + "user.bazelrc", + ], + ) + [ + "//backend:repo_go_test_srcs", + "//backend/conformance:repo_go_test_srcs", + "//beadserrors:repo_go_test_srcs", + "//cmd/bd:repo_go_test_srcs", + "//cmd/bd/doctor:repo_go_test_srcs", + "//cmd/bd/doctor/fix:repo_go_test_srcs", + "//cmd/bd/protocol:repo_go_test_srcs", + "//cmd/bd/setup:repo_go_test_srcs", + "//format:repo_go_test_srcs", + "//internal/ado:repo_go_test_srcs", + "//internal/atomicfile:repo_go_test_srcs", + "//internal/audit:repo_go_test_srcs", + "//internal/beads:repo_go_test_srcs", + "//internal/ceiling:repo_go_test_srcs", + "//internal/compact:repo_go_test_srcs", + "//internal/config:repo_go_test_srcs", + "//internal/configfile:repo_go_test_srcs", + "//internal/creds:repo_go_test_srcs", + "//internal/debug:repo_go_test_srcs", + "//internal/doltremote:repo_go_test_srcs", + "//internal/doltserver:repo_go_test_srcs", + "//internal/doltversion:repo_go_test_srcs", + "//internal/eventsjournal:repo_go_test_srcs", + "//internal/execenv:repo_go_test_srcs", + "//internal/fdhygiene:repo_go_test_srcs", + "//internal/formula:repo_go_test_srcs", + "//internal/formula/cmd/schemagen:repo_go_test_srcs", + "//internal/formula/schemagen:repo_go_test_srcs", + "//internal/git:repo_go_test_srcs", + "//internal/gitenv:repo_go_test_srcs", + "//internal/githooksenv:repo_go_test_srcs", + "//internal/github:repo_go_test_srcs", + "//internal/gitignore:repo_go_test_srcs", + "//internal/gitlab:repo_go_test_srcs", + "//internal/gittraceenv:repo_go_test_srcs", + "//internal/hooks:repo_go_test_srcs", + "//internal/httpapi:repo_go_test_srcs", + "//internal/httpapi/apigen:repo_go_test_srcs", + "//internal/httpapi/spec:repo_go_test_srcs", + "//internal/httpapi/wireshape:repo_go_test_srcs", + "//internal/httpapi/wireshape/cmd/gendigest:repo_go_test_srcs", + "//internal/httpclient:repo_go_test_srcs", + "//internal/httpclient/encode:repo_go_test_srcs", + "//internal/httpclient/wire:repo_go_test_srcs", + "//internal/idgen:repo_go_test_srcs", + "//internal/jira:repo_go_test_srcs", + "//internal/linear:repo_go_test_srcs", + "//internal/lockfile:repo_go_test_srcs", + "//internal/memoryapi:repo_go_test_srcs", + "//internal/metrics:repo_go_test_srcs", + "//internal/migration:repo_go_test_srcs", + "//internal/migration/legacysqlite:repo_go_test_srcs", + "//internal/molecules:repo_go_test_srcs", + "//internal/notion:repo_go_test_srcs", + "//internal/procid:repo_go_test_srcs", + "//internal/query:repo_go_test_srcs", + "//internal/recipes:repo_go_test_srcs", + "//internal/remotecache:repo_go_test_srcs", + "//internal/routing:repo_go_test_srcs", + "//internal/storage:repo_go_test_srcs", + "//internal/storage/backendnames:repo_go_test_srcs", + "//internal/storage/backends:repo_go_test_srcs", + "//internal/storage/batchbench:repo_go_test_srcs", + "//internal/storage/batchfixtures:repo_go_test_srcs", + "//internal/storage/contextinfo:repo_go_test_srcs", + "//internal/storage/createbatchequiv:repo_go_test_srcs", + "//internal/storage/dberrors:repo_go_test_srcs", + "//internal/storage/dbproxy/identity:repo_go_test_srcs", + "//internal/storage/dbproxy/pidfile:repo_go_test_srcs", + "//internal/storage/dbproxy/proxy:repo_go_test_srcs", + "//internal/storage/dbproxy/server:repo_go_test_srcs", + "//internal/storage/dbproxy/util:repo_go_test_srcs", + "//internal/storage/depid:repo_go_test_srcs", + "//internal/storage/dolt:repo_go_test_srcs", + "//internal/storage/doltutil:repo_go_test_srcs", + "//internal/storage/domain:repo_go_test_srcs", + "//internal/storage/domain/db:repo_go_test_srcs", + "//internal/storage/domain/fs:repo_go_test_srcs", + "//internal/storage/domain/git:repo_go_test_srcs", + "//internal/storage/embeddeddolt:repo_go_test_srcs", + "//internal/storage/embeddeddolt/cmd:repo_go_test_srcs", + "//internal/storage/externaldeps:repo_go_test_srcs", + "//internal/storage/fs:repo_go_test_srcs", + "//internal/storage/git:repo_go_test_srcs", + "//internal/storage/issueops:repo_go_test_srcs", + "//internal/storage/journalscan:repo_go_test_srcs", + "//internal/storage/kvkeys:repo_go_test_srcs", + "//internal/storage/memoryops:repo_go_test_srcs", + "//internal/storage/rowid:repo_go_test_srcs", + "//internal/storage/schema:repo_go_test_srcs", + "//internal/storage/sqlbuild:repo_go_test_srcs", + "//internal/storage/sqlclass:repo_go_test_srcs", + "//internal/storage/sqlcount:repo_go_test_srcs", + "//internal/storage/storagecontract:repo_go_test_srcs", + "//internal/storage/unsupportedgen:repo_go_test_srcs", + "//internal/storage/uow:repo_go_test_srcs", + "//internal/storage/versioncontrolops:repo_go_test_srcs", + "//internal/telemetry:repo_go_test_srcs", + "//internal/templates:repo_go_test_srcs", + "//internal/templates/agents:repo_go_test_srcs", + "//internal/testutil:repo_go_test_srcs", + "//internal/testutil/bazeltest:repo_go_test_srcs", + "//internal/testutil/credentialcmd:repo_go_test_srcs", + "//internal/testutil/fixtures:repo_go_test_srcs", + "//internal/testutil/integration:repo_go_test_srcs", + "//internal/timeparsing:repo_go_test_srcs", + "//internal/tracker:repo_go_test_srcs", + "//internal/tracker/conformance:repo_go_test_srcs", + "//internal/types:repo_go_test_srcs", + "//internal/ui:repo_go_test_srcs", + "//internal/uimd:repo_go_test_srcs", + "//internal/utils:repo_go_test_srcs", + "//internal/validation:repo_go_test_srcs", + "//internal/versioncheck:repo_go_test_srcs", + "//internal/workapi:repo_go_test_srcs", + "//internal/workapi/storecounter:repo_go_test_srcs", + "//internal/workapi/storequerier:repo_go_test_srcs", + "//internal/workapi/storereader:repo_go_test_srcs", + "//internal/workapi/storereadycounter:repo_go_test_srcs", + "//internal/workapi/storestats:repo_go_test_srcs", + "//internal/workapi/storeversionreconciler:repo_go_test_srcs", + "//internal/workapi/storeworkspaceconfig:repo_go_test_srcs", + "//internal/workspacegate:repo_go_test_srcs", + "//internal/worktreeremove:repo_go_test_srcs", + "//issueops:repo_go_test_srcs", + "//journalops:repo_go_test_srcs", + "//memoryops:repo_go_test_srcs", + "//platforms:repo_go_test_srcs", + "//plugins/beads:repo_go_test_srcs", + "//schema:repo_go_test_srcs", + "//scripts:repo_go_test_srcs", + "//scripts/bench-ready-indexes:repo_go_test_srcs", + "//scripts/check-versions:repo_go_test_srcs", + "//scripts/checkworkflowtags:repo_go_test_srcs", + "//scripts/gitattributespolicy:repo_go_test_srcs", + "//scripts/internal/testbash:repo_go_test_srcs", + "//scripts/migration-test/catalog:repo_go_test_srcs", + "//scripts/pr-lint:repo_go_test_srcs", + "//scripts/prlintmake:repo_go_test_srcs", + "//scripts/repochecks:repo_go_test_srcs", + "//scripts/repro-dolt-hang:repo_go_test_srcs", + "//scripts/repro-dolt-prod-timeouts:repo_go_test_srcs", + "//scripts/workflowtags:repo_go_test_srcs", + "//test/conformance:repo_go_test_srcs", + "//test/docsync:repo_go_test_srcs", + "//test/testmainconvention:repo_go_test_srcs", + "//tests/hook_timeout_backends:repo_go_test_srcs", + "//tests/migration:repo_go_test_srcs", + "//tests/regression:repo_go_test_srcs", + "//tests/upgrade_smoke:repo_go_test_srcs", + "//third_party/cc_toolchain:repo_go_test_srcs", + "//third_party/patches:repo_go_test_srcs", + "//tools/bazel:repo_go_test_srcs", + "//tools/docsmint:repo_go_test_srcs", + "//tools/nogo:repo_go_test_srcs", + "//tools/nogo/analyzers/depguard:repo_go_test_srcs", + "//tools/nogo/analyzers/errcheck:repo_go_test_srcs", + "//tools/nogo/analyzers/forbidigo:repo_go_test_srcs", + "//tools/nogo/analyzers/gosec:repo_go_test_srcs", + "//tools/nogo/analyzers/misspell:repo_go_test_srcs", + "//tools/nogo/analyzers/sloglint:repo_go_test_srcs", + "//tools/nogo/analyzers/unconvert:repo_go_test_srcs", + "//tools/nogo/analyzers/unparam:repo_go_test_srcs", + "//tools/nogo/internal/analyzertest:repo_go_test_srcs", + "//tools/nogo/internal/golangci:repo_go_test_srcs", + "//tools/nogo/internal/position:repo_go_test_srcs", + ], + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -251,178 +968,174 @@ filegroup( "user.bazelrc", ], ) + [ - "//backend:repo_files", - "//backend/conformance:repo_files", - "//beadserrors:repo_files", - "//cmd/bd:repo_files", - "//cmd/bd/doctor:repo_files", - "//cmd/bd/doctor/fix:repo_files", - "//cmd/bd/protocol:repo_files", - "//cmd/bd/setup:repo_files", - "//format:repo_files", - "//internal/ado:repo_files", - "//internal/atomicfile:repo_files", - "//internal/audit:repo_files", - "//internal/beads:repo_files", - "//internal/ceiling:repo_files", - "//internal/compact:repo_files", - "//internal/config:repo_files", - "//internal/configfile:repo_files", - "//internal/creds:repo_files", - "//internal/debug:repo_files", - "//internal/doltremote:repo_files", - "//internal/doltserver:repo_files", - "//internal/doltversion:repo_files", - "//internal/eventsjournal:repo_files", - "//internal/execenv:repo_files", - "//internal/fdhygiene:repo_files", - "//internal/formula:repo_files", - "//internal/formula/cmd/schemagen:repo_files", - "//internal/formula/schemagen:repo_files", - "//internal/git:repo_files", - "//internal/gitenv:repo_files", - "//internal/githooksenv:repo_files", - "//internal/github:repo_files", - "//internal/gitignore:repo_files", - "//internal/gitlab:repo_files", - "//internal/gittraceenv:repo_files", - "//internal/hooks:repo_files", - "//internal/httpapi:repo_files", - "//internal/httpapi/apigen:repo_files", - "//internal/httpapi/spec:repo_files", - "//internal/httpapi/wireshape:repo_files", - "//internal/httpapi/wireshape/cmd/gendigest:repo_files", - "//internal/httpclient:repo_files", - "//internal/httpclient/encode:repo_files", - "//internal/httpclient/wire:repo_files", - "//internal/idgen:repo_files", - "//internal/jira:repo_files", - "//internal/linear:repo_files", - "//internal/lockfile:repo_files", - "//internal/memoryapi:repo_files", - "//internal/metrics:repo_files", - "//internal/migration:repo_files", - "//internal/migration/legacysqlite:repo_files", - "//internal/molecules:repo_files", - "//internal/notion:repo_files", - "//internal/procid:repo_files", - "//internal/query:repo_files", - "//internal/recipes:repo_files", - "//internal/remotecache:repo_files", - "//internal/routing:repo_files", - "//internal/storage:repo_files", - "//internal/storage/backendnames:repo_files", - "//internal/storage/backends:repo_files", - "//internal/storage/batchbench:repo_files", - "//internal/storage/batchfixtures:repo_files", - "//internal/storage/contextinfo:repo_files", - "//internal/storage/createbatchequiv:repo_files", - "//internal/storage/dberrors:repo_files", - "//internal/storage/dbproxy/identity:repo_files", - "//internal/storage/dbproxy/pidfile:repo_files", - "//internal/storage/dbproxy/proxy:repo_files", - "//internal/storage/dbproxy/server:repo_files", - "//internal/storage/dbproxy/util:repo_files", - "//internal/storage/depid:repo_files", - "//internal/storage/dolt:repo_files", - "//internal/storage/doltutil:repo_files", - "//internal/storage/domain:repo_files", - "//internal/storage/domain/db:repo_files", - "//internal/storage/domain/fs:repo_files", - "//internal/storage/domain/git:repo_files", - "//internal/storage/embeddeddolt:repo_files", - "//internal/storage/embeddeddolt/cmd:repo_files", - "//internal/storage/externaldeps:repo_files", - "//internal/storage/fs:repo_files", - "//internal/storage/git:repo_files", - "//internal/storage/issueops:repo_files", - "//internal/storage/journalscan:repo_files", - "//internal/storage/kvkeys:repo_files", - "//internal/storage/memoryops:repo_files", - "//internal/storage/rowid:repo_files", - "//internal/storage/schema:repo_files", - "//internal/storage/sqlbuild:repo_files", - "//internal/storage/sqlclass:repo_files", - "//internal/storage/sqlcount:repo_files", - "//internal/storage/storagecontract:repo_files", - "//internal/storage/unsupportedgen:repo_files", - "//internal/storage/uow:repo_files", - "//internal/storage/versioncontrolops:repo_files", - "//internal/telemetry:repo_files", - "//internal/templates:repo_files", - "//internal/templates/agents:repo_files", - "//internal/testutil:repo_files", - "//internal/testutil/bazeltest:repo_files", - "//internal/testutil/credentialcmd:repo_files", - "//internal/testutil/fixtures:repo_files", - "//internal/testutil/integration:repo_files", - "//internal/timeparsing:repo_files", - "//internal/tracker:repo_files", - "//internal/tracker/conformance:repo_files", - "//internal/types:repo_files", - "//internal/ui:repo_files", - "//internal/uimd:repo_files", - "//internal/utils:repo_files", - "//internal/validation:repo_files", - "//internal/versioncheck:repo_files", - "//internal/workapi:repo_files", - "//internal/workapi/storecounter:repo_files", - "//internal/workapi/storequerier:repo_files", - "//internal/workapi/storereader:repo_files", - "//internal/workapi/storereadycounter:repo_files", - "//internal/workapi/storestats:repo_files", - "//internal/workapi/storeversionreconciler:repo_files", - "//internal/workapi/storeworkspaceconfig:repo_files", - "//internal/workspacegate:repo_files", - "//internal/worktreeremove:repo_files", - "//issueops:repo_files", - "//journalops:repo_files", - "//memoryops:repo_files", - "//platforms:repo_files", - "//plugins/beads:repo_files", - "//schema:repo_files", - "//scripts:repo_files", - "//scripts/bench-ready-indexes:repo_files", - "//scripts/check-versions:repo_files", - "//scripts/checkworkflowtags:repo_files", - "//scripts/gitattributespolicy:repo_files", - "//scripts/internal/testbash:repo_files", - "//scripts/migration-test/catalog:repo_files", - "//scripts/pr-lint:repo_files", - "//scripts/prlintmake:repo_files", - "//scripts/repochecks:repo_files", - "//scripts/repro-dolt-hang:repo_files", - "//scripts/repro-dolt-prod-timeouts:repo_files", - "//scripts/workflowtags:repo_files", - "//test/conformance:repo_files", - "//test/docsync:repo_files", - "//test/testmainconvention:repo_files", - "//tests/hook_timeout_backends:repo_files", - "//tests/migration:repo_files", - "//tests/regression:repo_files", - "//tests/upgrade_smoke:repo_files", - "//third_party/cc_toolchain:repo_files", - "//third_party/patches:repo_files", - "//tools/bazel:repo_files", - "//tools/docsmint:repo_files", - "//tools/nogo:repo_files", - "//tools/nogo/analyzers/depguard:repo_files", - "//tools/nogo/analyzers/errcheck:repo_files", - "//tools/nogo/analyzers/forbidigo:repo_files", - "//tools/nogo/analyzers/gosec:repo_files", - "//tools/nogo/analyzers/misspell:repo_files", - "//tools/nogo/analyzers/sloglint:repo_files", - "//tools/nogo/analyzers/unconvert:repo_files", - "//tools/nogo/analyzers/unparam:repo_files", - "//tools/nogo/internal/analyzertest:repo_files", - "//tools/nogo/internal/golangci:repo_files", - "//tools/nogo/internal/position:repo_files", - ], - visibility = [ - "//scripts:__pkg__", - "//scripts/repochecks:__pkg__", - "//test/docsync:__pkg__", + "//backend:repo_other_files", + "//backend/conformance:repo_other_files", + "//beadserrors:repo_other_files", + "//cmd/bd:repo_other_files", + "//cmd/bd/doctor:repo_other_files", + "//cmd/bd/doctor/fix:repo_other_files", + "//cmd/bd/protocol:repo_other_files", + "//cmd/bd/setup:repo_other_files", + "//format:repo_other_files", + "//internal/ado:repo_other_files", + "//internal/atomicfile:repo_other_files", + "//internal/audit:repo_other_files", + "//internal/beads:repo_other_files", + "//internal/ceiling:repo_other_files", + "//internal/compact:repo_other_files", + "//internal/config:repo_other_files", + "//internal/configfile:repo_other_files", + "//internal/creds:repo_other_files", + "//internal/debug:repo_other_files", + "//internal/doltremote:repo_other_files", + "//internal/doltserver:repo_other_files", + "//internal/doltversion:repo_other_files", + "//internal/eventsjournal:repo_other_files", + "//internal/execenv:repo_other_files", + "//internal/fdhygiene:repo_other_files", + "//internal/formula:repo_other_files", + "//internal/formula/cmd/schemagen:repo_other_files", + "//internal/formula/schemagen:repo_other_files", + "//internal/git:repo_other_files", + "//internal/gitenv:repo_other_files", + "//internal/githooksenv:repo_other_files", + "//internal/github:repo_other_files", + "//internal/gitignore:repo_other_files", + "//internal/gitlab:repo_other_files", + "//internal/gittraceenv:repo_other_files", + "//internal/hooks:repo_other_files", + "//internal/httpapi:repo_other_files", + "//internal/httpapi/apigen:repo_other_files", + "//internal/httpapi/spec:repo_other_files", + "//internal/httpapi/wireshape:repo_other_files", + "//internal/httpapi/wireshape/cmd/gendigest:repo_other_files", + "//internal/httpclient:repo_other_files", + "//internal/httpclient/encode:repo_other_files", + "//internal/httpclient/wire:repo_other_files", + "//internal/idgen:repo_other_files", + "//internal/jira:repo_other_files", + "//internal/linear:repo_other_files", + "//internal/lockfile:repo_other_files", + "//internal/memoryapi:repo_other_files", + "//internal/metrics:repo_other_files", + "//internal/migration:repo_other_files", + "//internal/migration/legacysqlite:repo_other_files", + "//internal/molecules:repo_other_files", + "//internal/notion:repo_other_files", + "//internal/procid:repo_other_files", + "//internal/query:repo_other_files", + "//internal/recipes:repo_other_files", + "//internal/remotecache:repo_other_files", + "//internal/routing:repo_other_files", + "//internal/storage:repo_other_files", + "//internal/storage/backendnames:repo_other_files", + "//internal/storage/backends:repo_other_files", + "//internal/storage/batchbench:repo_other_files", + "//internal/storage/batchfixtures:repo_other_files", + "//internal/storage/contextinfo:repo_other_files", + "//internal/storage/createbatchequiv:repo_other_files", + "//internal/storage/dberrors:repo_other_files", + "//internal/storage/dbproxy/identity:repo_other_files", + "//internal/storage/dbproxy/pidfile:repo_other_files", + "//internal/storage/dbproxy/proxy:repo_other_files", + "//internal/storage/dbproxy/server:repo_other_files", + "//internal/storage/dbproxy/util:repo_other_files", + "//internal/storage/depid:repo_other_files", + "//internal/storage/dolt:repo_other_files", + "//internal/storage/doltutil:repo_other_files", + "//internal/storage/domain:repo_other_files", + "//internal/storage/domain/db:repo_other_files", + "//internal/storage/domain/fs:repo_other_files", + "//internal/storage/domain/git:repo_other_files", + "//internal/storage/embeddeddolt:repo_other_files", + "//internal/storage/embeddeddolt/cmd:repo_other_files", + "//internal/storage/externaldeps:repo_other_files", + "//internal/storage/fs:repo_other_files", + "//internal/storage/git:repo_other_files", + "//internal/storage/issueops:repo_other_files", + "//internal/storage/journalscan:repo_other_files", + "//internal/storage/kvkeys:repo_other_files", + "//internal/storage/memoryops:repo_other_files", + "//internal/storage/rowid:repo_other_files", + "//internal/storage/schema:repo_other_files", + "//internal/storage/sqlbuild:repo_other_files", + "//internal/storage/sqlclass:repo_other_files", + "//internal/storage/sqlcount:repo_other_files", + "//internal/storage/storagecontract:repo_other_files", + "//internal/storage/unsupportedgen:repo_other_files", + "//internal/storage/uow:repo_other_files", + "//internal/storage/versioncontrolops:repo_other_files", + "//internal/telemetry:repo_other_files", + "//internal/templates:repo_other_files", + "//internal/templates/agents:repo_other_files", + "//internal/testutil:repo_other_files", + "//internal/testutil/bazeltest:repo_other_files", + "//internal/testutil/credentialcmd:repo_other_files", + "//internal/testutil/fixtures:repo_other_files", + "//internal/testutil/integration:repo_other_files", + "//internal/timeparsing:repo_other_files", + "//internal/tracker:repo_other_files", + "//internal/tracker/conformance:repo_other_files", + "//internal/types:repo_other_files", + "//internal/ui:repo_other_files", + "//internal/uimd:repo_other_files", + "//internal/utils:repo_other_files", + "//internal/validation:repo_other_files", + "//internal/versioncheck:repo_other_files", + "//internal/workapi:repo_other_files", + "//internal/workapi/storecounter:repo_other_files", + "//internal/workapi/storequerier:repo_other_files", + "//internal/workapi/storereader:repo_other_files", + "//internal/workapi/storereadycounter:repo_other_files", + "//internal/workapi/storestats:repo_other_files", + "//internal/workapi/storeversionreconciler:repo_other_files", + "//internal/workapi/storeworkspaceconfig:repo_other_files", + "//internal/workspacegate:repo_other_files", + "//internal/worktreeremove:repo_other_files", + "//issueops:repo_other_files", + "//journalops:repo_other_files", + "//memoryops:repo_other_files", + "//platforms:repo_other_files", + "//plugins/beads:repo_other_files", + "//schema:repo_other_files", + "//scripts:repo_other_files", + "//scripts/bench-ready-indexes:repo_other_files", + "//scripts/check-versions:repo_other_files", + "//scripts/checkworkflowtags:repo_other_files", + "//scripts/gitattributespolicy:repo_other_files", + "//scripts/internal/testbash:repo_other_files", + "//scripts/migration-test/catalog:repo_other_files", + "//scripts/pr-lint:repo_other_files", + "//scripts/prlintmake:repo_other_files", + "//scripts/repochecks:repo_other_files", + "//scripts/repro-dolt-hang:repo_other_files", + "//scripts/repro-dolt-prod-timeouts:repo_other_files", + "//scripts/workflowtags:repo_other_files", + "//test/conformance:repo_other_files", + "//test/docsync:repo_other_files", + "//test/testmainconvention:repo_other_files", + "//tests/hook_timeout_backends:repo_other_files", + "//tests/migration:repo_other_files", + "//tests/regression:repo_other_files", + "//tests/upgrade_smoke:repo_other_files", + "//third_party/cc_toolchain:repo_other_files", + "//third_party/patches:repo_other_files", + "//tools/bazel:repo_other_files", + "//tools/docsmint:repo_other_files", + "//tools/nogo:repo_other_files", + "//tools/nogo/analyzers/depguard:repo_other_files", + "//tools/nogo/analyzers/errcheck:repo_other_files", + "//tools/nogo/analyzers/forbidigo:repo_other_files", + "//tools/nogo/analyzers/gosec:repo_other_files", + "//tools/nogo/analyzers/misspell:repo_other_files", + "//tools/nogo/analyzers/sloglint:repo_other_files", + "//tools/nogo/analyzers/unconvert:repo_other_files", + "//tools/nogo/analyzers/unparam:repo_other_files", + "//tools/nogo/internal/analyzertest:repo_other_files", + "//tools/nogo/internal/golangci:repo_other_files", + "//tools/nogo/internal/position:repo_other_files", ], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/MODULE.bazel b/MODULE.bazel index 4daf1ae58..5fb24db7e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -289,7 +289,7 @@ dolt_release( version = "2.1.8", ) -# The tracked release formula, for //scripts:scripts_test and the +# The tracked release formula, for //scripts:shell_scripts_test and the # whole-checkout guard //scripts/repochecks:go_install_guidance_test. .beads/ # also holds live bd data and is in .bazelignore, so no main-repo target can # declare the formula; this local repository exposes only .beads/formulas. diff --git a/backend/BUILD.bazel b/backend/BUILD.bazel index efaf0d7b9..912285a3e 100644 --- a/backend/BUILD.bazel +++ b/backend/BUILD.bazel @@ -47,12 +47,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -61,7 +100,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index 3ff778c49..1f9632e79 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -115,12 +115,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -129,7 +168,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/beadserrors/BUILD.bazel b/beadserrors/BUILD.bazel index 51c339a62..42563e971 100644 --- a/beadserrors/BUILD.bazel +++ b/beadserrors/BUILD.bazel @@ -23,12 +23,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 5964f512a..f220e60a9 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1611,6 +1611,20 @@ sh_test( tags = ["dolt-server-cmd"], ) +# Source files the docs and agent instructions link to by path, which +# //test/docsync checks resolve. +exports_files( + [ + "capability_registry.go", + "doctor.go", + "init_safety_help.go", + "label.go", + "main.go", + "state.go", + ], + visibility = ["//test/docsync:__pkg__"], +) + # CLI flag registrations pinned by //internal/httpapi spec parity tests. exports_files( [ @@ -1648,12 +1662,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -1662,7 +1715,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/cmd/bd/doctor/BUILD.bazel b/cmd/bd/doctor/BUILD.bazel index cfd8abbdb..2187b5e1f 100644 --- a/cmd/bd/doctor/BUILD.bazel +++ b/cmd/bd/doctor/BUILD.bazel @@ -191,12 +191,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -205,7 +244,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel index af5e6d02b..fd7245c04 100644 --- a/cmd/bd/doctor/fix/BUILD.bazel +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -142,12 +142,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -156,7 +195,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index 04a4c35b6..76c1694b2 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -109,12 +109,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -123,7 +162,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/cmd/bd/setup/BUILD.bazel b/cmd/bd/setup/BUILD.bazel index c95870119..1ab24afee 100644 --- a/cmd/bd/setup/BUILD.bazel +++ b/cmd/bd/setup/BUILD.bazel @@ -64,12 +64,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -78,7 +117,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index fd3f54f72..09ebaa67f 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -919,11 +919,16 @@ Required` requires them to have run remotely and passed. (`PR Lint (native|windows|darwin)`) jobs are retired. - The repository policy tests (`./scripts/...`, including the D2 guards) and the tests that walk the checkout run only under Bazel, remotely: - `//scripts:scripts_test` and `//test/docsync:docsync_test` take - `//:repo_files` as data, the checkout as Bazel sees it (every tracked - file outside `.bazelignore`, aggregated from the `repo_files` block - `tools/bazel/go_srcs.py` keeps in every package; the BUILD sync step's - `make bazel-sync-check` fails on a package without it). The release + the `//scripts` go_tests and `//test/docsync:docsync_test`, each over + the part of the checkout it reads. `tools/bazel/go_srcs.py` keeps a + `repo_files` block in every package that partitions its files into + `repo_go_srcs`, `repo_go_test_srcs`, `repo_doc_files` and + `repo_other_files`, aggregated at the root; their union `//:repo_files` + is every tracked file outside `.bazelignore`, which only + `//scripts:tracked_files_test` and two repository guards read (the + BUILD sync step's `make bazel-sync-check` fails on a package without + the block). So a Go-only change re-runs no workflow policy test, and a + docs-only or workflow-only change no Go-source scan. The release formula under `.bazelignore`d `.beads/` comes in as `@beads_formulas`. Their former `go test` legs (`Go checks (scripts-test)` and `Go checks (allowlisted)`) are gone. @@ -1176,7 +1181,7 @@ changes. They gate through `BAZEL_TEST`; pinned by | fast-checks: migration hygiene checks A and B | `//scripts/repochecks:migration_hygiene_test` (`MIGRATION_HYGIENE_SCOPE=tree`) | | fast-checks: `make fmt-check` | `//scripts/repochecks:fmt_test` (the registered SDK's gofmt) | | PR Policy: `checkworkflowtags` | `//scripts/repochecks:workflow_tags_test` | -| PR Policy: `check-testing-short.sh` | `//scripts:scripts_test` (`TestCheckTestingShortPassesOnCleanRepoTree`) | +| PR Policy: `check-testing-short.sh` | `//scripts:go_sources_test` (`TestCheckTestingShortPassesOnCleanRepoTree`) | | PR Policy: workapi frontend boundary | `//scripts/repochecks:workapi_frontend_boundary_test` | | PR Policy: `make api-check` drift | `//scripts/repochecks:types_gen_drift_test`; its `go test ./internal/httpapi/...` half is the httpapi targets in the same lane | | PR Policy / check-doc-flags: `check-doc-freshness.sh` | `//scripts/repochecks:doc_freshness_test` | diff --git a/format/BUILD.bazel b/format/BUILD.bazel index 5d295c6c6..920b2d461 100644 --- a/format/BUILD.bazel +++ b/format/BUILD.bazel @@ -21,12 +21,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -35,7 +74,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/ado/BUILD.bazel b/internal/ado/BUILD.bazel index 6f94e9775..f21cd61ed 100644 --- a/internal/ado/BUILD.bazel +++ b/internal/ado/BUILD.bazel @@ -53,12 +53,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -67,7 +106,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/atomicfile/BUILD.bazel b/internal/atomicfile/BUILD.bazel index ed52ef940..ca5c51c3a 100644 --- a/internal/atomicfile/BUILD.bazel +++ b/internal/atomicfile/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/audit/BUILD.bazel b/internal/audit/BUILD.bazel index 1cd6e6352..c6cc782c5 100644 --- a/internal/audit/BUILD.bazel +++ b/internal/audit/BUILD.bazel @@ -23,12 +23,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/beads/BUILD.bazel b/internal/beads/BUILD.bazel index 27c6349c1..f231d1a90 100644 --- a/internal/beads/BUILD.bazel +++ b/internal/beads/BUILD.bazel @@ -61,12 +61,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -75,7 +114,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/ceiling/BUILD.bazel b/internal/ceiling/BUILD.bazel index 7de0ab073..bd1679cf3 100644 --- a/internal/ceiling/BUILD.bazel +++ b/internal/ceiling/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/compact/BUILD.bazel b/internal/compact/BUILD.bazel index 3ff08a678..412850425 100644 --- a/internal/compact/BUILD.bazel +++ b/internal/compact/BUILD.bazel @@ -44,12 +44,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -58,7 +97,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/config/BUILD.bazel b/internal/config/BUILD.bazel index f2c2496a1..d3f96ab1e 100644 --- a/internal/config/BUILD.bazel +++ b/internal/config/BUILD.bazel @@ -71,12 +71,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -85,7 +124,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/configfile/BUILD.bazel b/internal/configfile/BUILD.bazel index 0d67a72a5..8e3dc50ba 100644 --- a/internal/configfile/BUILD.bazel +++ b/internal/configfile/BUILD.bazel @@ -36,12 +36,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -50,7 +89,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/creds/BUILD.bazel b/internal/creds/BUILD.bazel index ad796d4c1..950107cc8 100644 --- a/internal/creds/BUILD.bazel +++ b/internal/creds/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/debug/BUILD.bazel b/internal/debug/BUILD.bazel index f23f1e5c8..41622cf53 100644 --- a/internal/debug/BUILD.bazel +++ b/internal/debug/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/doltremote/BUILD.bazel b/internal/doltremote/BUILD.bazel index d0299f657..071ef320d 100644 --- a/internal/doltremote/BUILD.bazel +++ b/internal/doltremote/BUILD.bazel @@ -23,12 +23,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/doltserver/BUILD.bazel b/internal/doltserver/BUILD.bazel index 7fd648465..c1a3fe5c9 100644 --- a/internal/doltserver/BUILD.bazel +++ b/internal/doltserver/BUILD.bazel @@ -197,12 +197,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -211,7 +250,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/doltversion/BUILD.bazel b/internal/doltversion/BUILD.bazel index d49ef74b1..b4a71ebc9 100644 --- a/internal/doltversion/BUILD.bazel +++ b/internal/doltversion/BUILD.bazel @@ -30,12 +30,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/eventsjournal/BUILD.bazel b/internal/eventsjournal/BUILD.bazel index 2b16a7c91..f2f9f8d85 100644 --- a/internal/eventsjournal/BUILD.bazel +++ b/internal/eventsjournal/BUILD.bazel @@ -33,12 +33,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -47,7 +86,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/execenv/BUILD.bazel b/internal/execenv/BUILD.bazel index 7ab55c24f..583409cf8 100644 --- a/internal/execenv/BUILD.bazel +++ b/internal/execenv/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/fdhygiene/BUILD.bazel b/internal/fdhygiene/BUILD.bazel index ef2066d5a..388486b06 100644 --- a/internal/fdhygiene/BUILD.bazel +++ b/internal/fdhygiene/BUILD.bazel @@ -92,12 +92,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -106,7 +145,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/formula/BUILD.bazel b/internal/formula/BUILD.bazel index a7377a78d..7eaf0ac9c 100644 --- a/internal/formula/BUILD.bazel +++ b/internal/formula/BUILD.bazel @@ -55,12 +55,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -69,7 +108,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/formula/cmd/schemagen/BUILD.bazel b/internal/formula/cmd/schemagen/BUILD.bazel index aa9497029..65099f5f3 100644 --- a/internal/formula/cmd/schemagen/BUILD.bazel +++ b/internal/formula/cmd/schemagen/BUILD.bazel @@ -17,12 +17,51 @@ go_binary( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/formula/schemagen/BUILD.bazel b/internal/formula/schemagen/BUILD.bazel index fdcbe2f97..d37f04761 100644 --- a/internal/formula/schemagen/BUILD.bazel +++ b/internal/formula/schemagen/BUILD.bazel @@ -10,12 +10,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -24,7 +63,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/git/BUILD.bazel b/internal/git/BUILD.bazel index e36e0ec55..3a7da4a9d 100644 --- a/internal/git/BUILD.bazel +++ b/internal/git/BUILD.bazel @@ -25,12 +25,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -39,7 +78,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/gitenv/BUILD.bazel b/internal/gitenv/BUILD.bazel index b9317dac6..d87d84ee6 100644 --- a/internal/gitenv/BUILD.bazel +++ b/internal/gitenv/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/githooksenv/BUILD.bazel b/internal/githooksenv/BUILD.bazel index 0dfbf2c9b..ef95ce5a2 100644 --- a/internal/githooksenv/BUILD.bazel +++ b/internal/githooksenv/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/github/BUILD.bazel b/internal/github/BUILD.bazel index 8062dd22a..309a1d401 100644 --- a/internal/github/BUILD.bazel +++ b/internal/github/BUILD.bazel @@ -42,12 +42,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -56,7 +95,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/gitignore/BUILD.bazel b/internal/gitignore/BUILD.bazel index 3ec14028e..1f3a63467 100644 --- a/internal/gitignore/BUILD.bazel +++ b/internal/gitignore/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/gitlab/BUILD.bazel b/internal/gitlab/BUILD.bazel index 1ae60e524..dfdb4d11b 100644 --- a/internal/gitlab/BUILD.bazel +++ b/internal/gitlab/BUILD.bazel @@ -41,12 +41,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -55,7 +94,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/gittraceenv/BUILD.bazel b/internal/gittraceenv/BUILD.bazel index d3ab9ec8e..f5f7b10a2 100644 --- a/internal/gittraceenv/BUILD.bazel +++ b/internal/gittraceenv/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/hooks/BUILD.bazel b/internal/hooks/BUILD.bazel index 3c6e1590b..f389164a2 100644 --- a/internal/hooks/BUILD.bazel +++ b/internal/hooks/BUILD.bazel @@ -217,12 +217,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -231,7 +270,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index ff7bb7787..fb7b25625 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -134,12 +134,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -148,7 +187,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpapi/apigen/BUILD.bazel b/internal/httpapi/apigen/BUILD.bazel index 9aa4a0b29..fe6159f34 100644 --- a/internal/httpapi/apigen/BUILD.bazel +++ b/internal/httpapi/apigen/BUILD.bazel @@ -25,12 +25,51 @@ exports_files( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -39,7 +78,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpapi/spec/BUILD.bazel b/internal/httpapi/spec/BUILD.bazel index 81d0f6532..51f81d945 100644 --- a/internal/httpapi/spec/BUILD.bazel +++ b/internal/httpapi/spec/BUILD.bazel @@ -26,12 +26,51 @@ exports_files( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -40,7 +79,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpapi/wireshape/BUILD.bazel b/internal/httpapi/wireshape/BUILD.bazel index c107b8a0c..6992e365b 100644 --- a/internal/httpapi/wireshape/BUILD.bazel +++ b/internal/httpapi/wireshape/BUILD.bazel @@ -38,12 +38,51 @@ exports_files( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -52,7 +91,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel b/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel index 3b3dc5d04..ac908f90e 100644 --- a/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel +++ b/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel @@ -27,12 +27,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index f6cb1cbee..beffe7048 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -253,12 +253,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -267,7 +306,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpclient/encode/BUILD.bazel b/internal/httpclient/encode/BUILD.bazel index 1b0e8cee7..f09bdd196 100644 --- a/internal/httpclient/encode/BUILD.bazel +++ b/internal/httpclient/encode/BUILD.bazel @@ -51,12 +51,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -65,7 +104,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpclient/wire/BUILD.bazel b/internal/httpclient/wire/BUILD.bazel index e154bc59f..012a18995 100644 --- a/internal/httpclient/wire/BUILD.bazel +++ b/internal/httpclient/wire/BUILD.bazel @@ -63,12 +63,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -77,7 +116,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/idgen/BUILD.bazel b/internal/idgen/BUILD.bazel index d85848c47..1f6527be1 100644 --- a/internal/idgen/BUILD.bazel +++ b/internal/idgen/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/jira/BUILD.bazel b/internal/jira/BUILD.bazel index f42e6ec5e..ee2f37feb 100644 --- a/internal/jira/BUILD.bazel +++ b/internal/jira/BUILD.bazel @@ -41,12 +41,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -55,7 +94,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/linear/BUILD.bazel b/internal/linear/BUILD.bazel index e660b46a2..f384b1cba 100644 --- a/internal/linear/BUILD.bazel +++ b/internal/linear/BUILD.bazel @@ -67,12 +67,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -81,7 +120,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/lockfile/BUILD.bazel b/internal/lockfile/BUILD.bazel index 347cd13d9..213cb7fb1 100644 --- a/internal/lockfile/BUILD.bazel +++ b/internal/lockfile/BUILD.bazel @@ -63,12 +63,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -77,7 +116,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/memoryapi/BUILD.bazel b/internal/memoryapi/BUILD.bazel index d676833ae..3636ccb1b 100644 --- a/internal/memoryapi/BUILD.bazel +++ b/internal/memoryapi/BUILD.bazel @@ -18,12 +18,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -32,7 +71,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/metrics/BUILD.bazel b/internal/metrics/BUILD.bazel index eb9b7d7fa..275e4ce8f 100644 --- a/internal/metrics/BUILD.bazel +++ b/internal/metrics/BUILD.bazel @@ -42,12 +42,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -56,7 +95,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/migration/BUILD.bazel b/internal/migration/BUILD.bazel index d3bae9093..be6405b19 100644 --- a/internal/migration/BUILD.bazel +++ b/internal/migration/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/migration/legacysqlite/BUILD.bazel b/internal/migration/legacysqlite/BUILD.bazel index 847117321..e7c57ad30 100644 --- a/internal/migration/legacysqlite/BUILD.bazel +++ b/internal/migration/legacysqlite/BUILD.bazel @@ -29,12 +29,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/molecules/BUILD.bazel b/internal/molecules/BUILD.bazel index 57fe6e3eb..5c63272a3 100644 --- a/internal/molecules/BUILD.bazel +++ b/internal/molecules/BUILD.bazel @@ -30,12 +30,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/notion/BUILD.bazel b/internal/notion/BUILD.bazel index 871fe68e8..d677dd138 100644 --- a/internal/notion/BUILD.bazel +++ b/internal/notion/BUILD.bazel @@ -40,12 +40,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -54,7 +93,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/procid/BUILD.bazel b/internal/procid/BUILD.bazel index 14cc92d61..908f7dc0f 100644 --- a/internal/procid/BUILD.bazel +++ b/internal/procid/BUILD.bazel @@ -53,12 +53,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -67,7 +106,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/query/BUILD.bazel b/internal/query/BUILD.bazel index d8db3bcff..81bcc479d 100644 --- a/internal/query/BUILD.bazel +++ b/internal/query/BUILD.bazel @@ -30,12 +30,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/recipes/BUILD.bazel b/internal/recipes/BUILD.bazel index b94f373c0..b7bc2c92b 100644 --- a/internal/recipes/BUILD.bazel +++ b/internal/recipes/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/remotecache/BUILD.bazel b/internal/remotecache/BUILD.bazel index 69d123c65..5f72b01ea 100644 --- a/internal/remotecache/BUILD.bazel +++ b/internal/remotecache/BUILD.bazel @@ -32,12 +32,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -46,7 +85,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/routing/BUILD.bazel b/internal/routing/BUILD.bazel index 87bee6e4d..4d0da09a2 100644 --- a/internal/routing/BUILD.bazel +++ b/internal/routing/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index e8ed03a5c..86a1ef4f6 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -182,12 +182,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -196,7 +235,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/backendnames/BUILD.bazel b/internal/storage/backendnames/BUILD.bazel index 90aa5c611..cde019f6b 100644 --- a/internal/storage/backendnames/BUILD.bazel +++ b/internal/storage/backendnames/BUILD.bazel @@ -23,12 +23,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/backends/BUILD.bazel b/internal/storage/backends/BUILD.bazel index d5c06c542..6d8a7ce76 100644 --- a/internal/storage/backends/BUILD.bazel +++ b/internal/storage/backends/BUILD.bazel @@ -42,12 +42,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -56,7 +95,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/batchbench/BUILD.bazel b/internal/storage/batchbench/BUILD.bazel index 7b15e3ec5..a689d08c2 100644 --- a/internal/storage/batchbench/BUILD.bazel +++ b/internal/storage/batchbench/BUILD.bazel @@ -31,12 +31,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -45,7 +84,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/batchfixtures/BUILD.bazel b/internal/storage/batchfixtures/BUILD.bazel index a4718d9bf..cfeac392a 100644 --- a/internal/storage/batchfixtures/BUILD.bazel +++ b/internal/storage/batchfixtures/BUILD.bazel @@ -43,12 +43,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -57,7 +96,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/contextinfo/BUILD.bazel b/internal/storage/contextinfo/BUILD.bazel index 00814c404..031802ba0 100644 --- a/internal/storage/contextinfo/BUILD.bazel +++ b/internal/storage/contextinfo/BUILD.bazel @@ -33,12 +33,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -47,7 +86,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/createbatchequiv/BUILD.bazel b/internal/storage/createbatchequiv/BUILD.bazel index 39cb683ac..4d76b4ee5 100644 --- a/internal/storage/createbatchequiv/BUILD.bazel +++ b/internal/storage/createbatchequiv/BUILD.bazel @@ -43,12 +43,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -57,7 +96,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dberrors/BUILD.bazel b/internal/storage/dberrors/BUILD.bazel index 5a5614ffe..4e0b623f7 100644 --- a/internal/storage/dberrors/BUILD.bazel +++ b/internal/storage/dberrors/BUILD.bazel @@ -37,12 +37,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -51,7 +90,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dbproxy/identity/BUILD.bazel b/internal/storage/dbproxy/identity/BUILD.bazel index de7e8243c..6875fd2fe 100644 --- a/internal/storage/dbproxy/identity/BUILD.bazel +++ b/internal/storage/dbproxy/identity/BUILD.bazel @@ -37,12 +37,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -51,7 +90,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dbproxy/pidfile/BUILD.bazel b/internal/storage/dbproxy/pidfile/BUILD.bazel index 47e4de2d1..294d2e682 100644 --- a/internal/storage/dbproxy/pidfile/BUILD.bazel +++ b/internal/storage/dbproxy/pidfile/BUILD.bazel @@ -30,12 +30,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dbproxy/proxy/BUILD.bazel b/internal/storage/dbproxy/proxy/BUILD.bazel index f3691c871..28d21fa02 100644 --- a/internal/storage/dbproxy/proxy/BUILD.bazel +++ b/internal/storage/dbproxy/proxy/BUILD.bazel @@ -112,12 +112,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -126,7 +165,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dbproxy/server/BUILD.bazel b/internal/storage/dbproxy/server/BUILD.bazel index 4236238be..151f3ac5a 100644 --- a/internal/storage/dbproxy/server/BUILD.bazel +++ b/internal/storage/dbproxy/server/BUILD.bazel @@ -66,12 +66,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -80,7 +119,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dbproxy/util/BUILD.bazel b/internal/storage/dbproxy/util/BUILD.bazel index e0a65d2f5..1efad1bf3 100644 --- a/internal/storage/dbproxy/util/BUILD.bazel +++ b/internal/storage/dbproxy/util/BUILD.bazel @@ -37,12 +37,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -51,7 +90,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/depid/BUILD.bazel b/internal/storage/depid/BUILD.bazel index 6fa6f9f05..1f6bafa88 100644 --- a/internal/storage/depid/BUILD.bazel +++ b/internal/storage/depid/BUILD.bazel @@ -30,12 +30,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 987879f40..989f2d8e4 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -596,12 +596,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -610,7 +649,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/doltutil/BUILD.bazel b/internal/storage/doltutil/BUILD.bazel index d018dd26b..3dae2c8b2 100644 --- a/internal/storage/doltutil/BUILD.bazel +++ b/internal/storage/doltutil/BUILD.bazel @@ -59,12 +59,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -73,7 +112,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/domain/BUILD.bazel b/internal/storage/domain/BUILD.bazel index 4393d650a..691757192 100644 --- a/internal/storage/domain/BUILD.bazel +++ b/internal/storage/domain/BUILD.bazel @@ -93,12 +93,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -107,7 +146,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index 823ed995a..0b7df5abb 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -168,12 +168,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -182,7 +221,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/domain/fs/BUILD.bazel b/internal/storage/domain/fs/BUILD.bazel index 4d6d2e8cb..2ce97fd3f 100644 --- a/internal/storage/domain/fs/BUILD.bazel +++ b/internal/storage/domain/fs/BUILD.bazel @@ -81,12 +81,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -95,7 +134,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/domain/git/BUILD.bazel b/internal/storage/domain/git/BUILD.bazel index 7b4b8f2e0..d2c13e895 100644 --- a/internal/storage/domain/git/BUILD.bazel +++ b/internal/storage/domain/git/BUILD.bazel @@ -68,12 +68,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -82,7 +121,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 181e37e32..e4e9258db 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -422,12 +422,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -436,7 +475,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/embeddeddolt/cmd/BUILD.bazel b/internal/storage/embeddeddolt/cmd/BUILD.bazel index 4fa70405d..8a9136e0b 100644 --- a/internal/storage/embeddeddolt/cmd/BUILD.bazel +++ b/internal/storage/embeddeddolt/cmd/BUILD.bazel @@ -38,12 +38,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -52,7 +91,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/externaldeps/BUILD.bazel b/internal/storage/externaldeps/BUILD.bazel index d558c90a4..31715e16c 100644 --- a/internal/storage/externaldeps/BUILD.bazel +++ b/internal/storage/externaldeps/BUILD.bazel @@ -59,12 +59,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -73,7 +112,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/fs/BUILD.bazel b/internal/storage/fs/BUILD.bazel index 4ef61a465..48994986c 100644 --- a/internal/storage/fs/BUILD.bazel +++ b/internal/storage/fs/BUILD.bazel @@ -27,12 +27,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/git/BUILD.bazel b/internal/storage/git/BUILD.bazel index 25e548a7b..5ede35634 100644 --- a/internal/storage/git/BUILD.bazel +++ b/internal/storage/git/BUILD.bazel @@ -27,12 +27,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index af5f9fc2e..0838bce48 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -234,12 +234,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -248,7 +287,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/journalscan/BUILD.bazel b/internal/storage/journalscan/BUILD.bazel index d1178ba0e..0b829365e 100644 --- a/internal/storage/journalscan/BUILD.bazel +++ b/internal/storage/journalscan/BUILD.bazel @@ -29,12 +29,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/kvkeys/BUILD.bazel b/internal/storage/kvkeys/BUILD.bazel index ee4c8e5cd..0c82c0ad2 100644 --- a/internal/storage/kvkeys/BUILD.bazel +++ b/internal/storage/kvkeys/BUILD.bazel @@ -29,12 +29,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/memoryops/BUILD.bazel b/internal/storage/memoryops/BUILD.bazel index 9039a52f6..19b7f8a47 100644 --- a/internal/storage/memoryops/BUILD.bazel +++ b/internal/storage/memoryops/BUILD.bazel @@ -33,12 +33,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -47,7 +86,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/rowid/BUILD.bazel b/internal/storage/rowid/BUILD.bazel index 90788b3c9..b0f65270f 100644 --- a/internal/storage/rowid/BUILD.bazel +++ b/internal/storage/rowid/BUILD.bazel @@ -30,12 +30,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index 7d663e2e7..57e38358c 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -198,12 +198,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -212,7 +251,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/sqlbuild/BUILD.bazel b/internal/storage/sqlbuild/BUILD.bazel index 8f2ce0aa2..7ff089896 100644 --- a/internal/storage/sqlbuild/BUILD.bazel +++ b/internal/storage/sqlbuild/BUILD.bazel @@ -56,12 +56,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -70,7 +109,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/sqlclass/BUILD.bazel b/internal/storage/sqlclass/BUILD.bazel index f169dff0d..3682f5ad7 100644 --- a/internal/storage/sqlclass/BUILD.bazel +++ b/internal/storage/sqlclass/BUILD.bazel @@ -30,12 +30,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/sqlcount/BUILD.bazel b/internal/storage/sqlcount/BUILD.bazel index eaa94c1b4..3abb00ede 100644 --- a/internal/storage/sqlcount/BUILD.bazel +++ b/internal/storage/sqlcount/BUILD.bazel @@ -23,12 +23,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/storagecontract/BUILD.bazel b/internal/storage/storagecontract/BUILD.bazel index 416cedece..b8e628c32 100644 --- a/internal/storage/storagecontract/BUILD.bazel +++ b/internal/storage/storagecontract/BUILD.bazel @@ -24,12 +24,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/unsupportedgen/BUILD.bazel b/internal/storage/unsupportedgen/BUILD.bazel index 9732d963a..0a9b6c0d5 100644 --- a/internal/storage/unsupportedgen/BUILD.bazel +++ b/internal/storage/unsupportedgen/BUILD.bazel @@ -37,12 +37,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -51,7 +90,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index d98cfcc61..8b4abc0d4 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -250,12 +250,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -264,7 +303,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/versioncontrolops/BUILD.bazel b/internal/storage/versioncontrolops/BUILD.bazel index 17fc465ad..84fcf6342 100644 --- a/internal/storage/versioncontrolops/BUILD.bazel +++ b/internal/storage/versioncontrolops/BUILD.bazel @@ -69,12 +69,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -83,7 +122,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel index 380b9d29a..90792516b 100644 --- a/internal/telemetry/BUILD.bazel +++ b/internal/telemetry/BUILD.bazel @@ -87,12 +87,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -101,7 +140,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/templates/BUILD.bazel b/internal/templates/BUILD.bazel index 521b4505e..799491071 100644 --- a/internal/templates/BUILD.bazel +++ b/internal/templates/BUILD.bazel @@ -14,12 +14,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -28,7 +67,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/templates/agents/BUILD.bazel b/internal/templates/agents/BUILD.bazel index 4836ed286..c51ae9c90 100644 --- a/internal/templates/agents/BUILD.bazel +++ b/internal/templates/agents/BUILD.bazel @@ -28,12 +28,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -42,7 +81,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 5fef41c5a..03563cbde 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -154,15 +154,64 @@ sh_test( tags = ["dolt-server"], ) +# The pinned dolt sql-server image, read by //scripts:scripts_test (the CI +# pins agree) and //test/docsync (the docs' pin agrees). +exports_files( + ["testdoltcommon.go"], + visibility = [ + "//scripts:__pkg__", + "//test/docsync:__pkg__", + ], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -171,7 +220,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/testutil/bazeltest/BUILD.bazel b/internal/testutil/bazeltest/BUILD.bazel index a3a6c955d..cdb9d6d9e 100644 --- a/internal/testutil/bazeltest/BUILD.bazel +++ b/internal/testutil/bazeltest/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/testutil/credentialcmd/BUILD.bazel b/internal/testutil/credentialcmd/BUILD.bazel index 82e67577d..d37bbe4a3 100644 --- a/internal/testutil/credentialcmd/BUILD.bazel +++ b/internal/testutil/credentialcmd/BUILD.bazel @@ -20,12 +20,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -34,7 +73,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/testutil/fixtures/BUILD.bazel b/internal/testutil/fixtures/BUILD.bazel index aee139035..b8b65f00d 100644 --- a/internal/testutil/fixtures/BUILD.bazel +++ b/internal/testutil/fixtures/BUILD.bazel @@ -29,12 +29,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/testutil/integration/BUILD.bazel b/internal/testutil/integration/BUILD.bazel index fcc3fc611..40d2017d5 100644 --- a/internal/testutil/integration/BUILD.bazel +++ b/internal/testutil/integration/BUILD.bazel @@ -83,12 +83,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -97,7 +136,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/timeparsing/BUILD.bazel b/internal/timeparsing/BUILD.bazel index 200d3d297..c9d379697 100644 --- a/internal/timeparsing/BUILD.bazel +++ b/internal/timeparsing/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/tracker/BUILD.bazel b/internal/tracker/BUILD.bazel index 1fb51af6d..853720cdd 100644 --- a/internal/tracker/BUILD.bazel +++ b/internal/tracker/BUILD.bazel @@ -89,12 +89,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -103,7 +142,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/tracker/conformance/BUILD.bazel b/internal/tracker/conformance/BUILD.bazel index 660c18c85..87299f0ae 100644 --- a/internal/tracker/conformance/BUILD.bazel +++ b/internal/tracker/conformance/BUILD.bazel @@ -55,12 +55,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -69,7 +108,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/types/BUILD.bazel b/internal/types/BUILD.bazel index d81695be6..ff7b6c8eb 100644 --- a/internal/types/BUILD.bazel +++ b/internal/types/BUILD.bazel @@ -44,12 +44,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -58,7 +97,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/ui/BUILD.bazel b/internal/ui/BUILD.bazel index d4bf16eb4..78e070e78 100644 --- a/internal/ui/BUILD.bazel +++ b/internal/ui/BUILD.bazel @@ -35,12 +35,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -49,7 +88,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/uimd/BUILD.bazel b/internal/uimd/BUILD.bazel index 756358b1c..dbbb4fffa 100644 --- a/internal/uimd/BUILD.bazel +++ b/internal/uimd/BUILD.bazel @@ -27,12 +27,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/utils/BUILD.bazel b/internal/utils/BUILD.bazel index be3234d57..ebc93a29a 100644 --- a/internal/utils/BUILD.bazel +++ b/internal/utils/BUILD.bazel @@ -53,12 +53,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -67,7 +106,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/validation/BUILD.bazel b/internal/validation/BUILD.bazel index 53e2232aa..5b75f8787 100644 --- a/internal/validation/BUILD.bazel +++ b/internal/validation/BUILD.bazel @@ -34,12 +34,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -48,7 +87,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/versioncheck/BUILD.bazel b/internal/versioncheck/BUILD.bazel index ffa55c8ad..0a7962628 100644 --- a/internal/versioncheck/BUILD.bazel +++ b/internal/versioncheck/BUILD.bazel @@ -22,12 +22,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -36,7 +75,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/BUILD.bazel b/internal/workapi/BUILD.bazel index ff896671c..89ecb128a 100644 --- a/internal/workapi/BUILD.bazel +++ b/internal/workapi/BUILD.bazel @@ -77,12 +77,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -91,7 +130,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storecounter/BUILD.bazel b/internal/workapi/storecounter/BUILD.bazel index 04b67721e..05bb55bb2 100644 --- a/internal/workapi/storecounter/BUILD.bazel +++ b/internal/workapi/storecounter/BUILD.bazel @@ -16,12 +16,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storequerier/BUILD.bazel b/internal/workapi/storequerier/BUILD.bazel index a8d2e6149..67b7a29b9 100644 --- a/internal/workapi/storequerier/BUILD.bazel +++ b/internal/workapi/storequerier/BUILD.bazel @@ -15,12 +15,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -29,7 +68,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storereader/BUILD.bazel b/internal/workapi/storereader/BUILD.bazel index 84e40c33e..5b0c187e6 100644 --- a/internal/workapi/storereader/BUILD.bazel +++ b/internal/workapi/storereader/BUILD.bazel @@ -27,12 +27,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storereadycounter/BUILD.bazel b/internal/workapi/storereadycounter/BUILD.bazel index 6cde32faf..29c4998ef 100644 --- a/internal/workapi/storereadycounter/BUILD.bazel +++ b/internal/workapi/storereadycounter/BUILD.bazel @@ -15,12 +15,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -29,7 +68,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storestats/BUILD.bazel b/internal/workapi/storestats/BUILD.bazel index 7b009776b..d0b08a598 100644 --- a/internal/workapi/storestats/BUILD.bazel +++ b/internal/workapi/storestats/BUILD.bazel @@ -16,12 +16,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storeversionreconciler/BUILD.bazel b/internal/workapi/storeversionreconciler/BUILD.bazel index db79c94b7..9eac2b29b 100644 --- a/internal/workapi/storeversionreconciler/BUILD.bazel +++ b/internal/workapi/storeversionreconciler/BUILD.bazel @@ -15,12 +15,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -29,7 +68,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storeworkspaceconfig/BUILD.bazel b/internal/workapi/storeworkspaceconfig/BUILD.bazel index c65c1f592..166762ce9 100644 --- a/internal/workapi/storeworkspaceconfig/BUILD.bazel +++ b/internal/workapi/storeworkspaceconfig/BUILD.bazel @@ -15,12 +15,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -29,7 +68,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workspacegate/BUILD.bazel b/internal/workspacegate/BUILD.bazel index db9bf9acf..85a5fefdf 100644 --- a/internal/workspacegate/BUILD.bazel +++ b/internal/workspacegate/BUILD.bazel @@ -18,12 +18,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -32,7 +71,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/worktreeremove/BUILD.bazel b/internal/worktreeremove/BUILD.bazel index 97c70cf13..9713ee55b 100644 --- a/internal/worktreeremove/BUILD.bazel +++ b/internal/worktreeremove/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index 1878a4e17..dc28cd166 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -72,12 +72,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -86,7 +125,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/journalops/BUILD.bazel b/journalops/BUILD.bazel index e42f82468..5449e5601 100644 --- a/journalops/BUILD.bazel +++ b/journalops/BUILD.bazel @@ -27,12 +27,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/memoryops/BUILD.bazel b/memoryops/BUILD.bazel index be45b9fa1..fb622d6a2 100644 --- a/memoryops/BUILD.bazel +++ b/memoryops/BUILD.bazel @@ -28,12 +28,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -42,7 +81,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 35cd55d7e..2707bcd42 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -32,12 +32,51 @@ platform( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -46,7 +85,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/plugins/beads/BUILD.bazel b/plugins/beads/BUILD.bazel index 618b55c28..feed1f222 100644 --- a/plugins/beads/BUILD.bazel +++ b/plugins/beads/BUILD.bazel @@ -35,12 +35,51 @@ exports_files( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -49,7 +88,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/schema/BUILD.bazel b/schema/BUILD.bazel index 08eae5b47..ccf861bdb 100644 --- a/schema/BUILD.bazel +++ b/schema/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index c7ba45203..92a065548 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -2,8 +2,22 @@ load("@rules_go//go:def.bzl", "go_test") # CI and repo-policy tests: they parse the workflows, run the CI shell scripts # against fixtures, and check repository-wide rules (BUILD tag taxonomy, skip -# allowlist, managed BUILD blocks, shard manifests, tracked-file scans) over -# //:repo_files, the checkout as Bazel sees it. +# allowlist, managed BUILD blocks, shard manifests, tracked-file scans). +# +# They are one Go package but several go_test targets, each compiling only +# the files whose tests it runs and declaring only what those tests read, so +# that a change re-runs only the targets that read it: a Go-only change runs +# no workflow policy test, a docs-only or workflow-only change no Go-source +# scan. The data are the partitions of the checkout that tools/bazel/go_srcs.py +# generates (//:repo_go_srcs, //:repo_go_test_srcs, //:repo_doc_files, +# //:repo_other_files, and per package) or single files; only +# :tracked_files_test declares //:repo_files, every tracked file. +# +# Every *_test.go here is in exactly one of these targets, except the +# helpers-only files (no Test functions) that several compile; srcs are kept +# by hand (`# keep` stops gazelle from putting every file back into +# scripts_test), and TestScriptsTestTargetsPartitionTheTests +# (scripts_targets_test.go) fails on a file in no target or a test in two. # # No tags: they run remotely like any other target. The scripts and fixtures # run bash, git, python3, jq, yq, curl, openssl, unzip and coreutils from the @@ -11,6 +25,11 @@ load("@rules_go//go:def.bzl", "go_test") # missing tool fails the test under Bazel instead of skipping it # (host_tools_test.go), so a cached pass never stands for an executor that # could not run it. Go and the release formula come in hermetically. + +# The CI and Bazel policy tests: workflows, the setup-bazel action, .bazelrc, +# MODULE.bazel, every BUILD and .bzl file, tools/bazel and the CI scripts, +# all in //:repo_other_files (every tracked file but Go and Markdown source), +# plus the one Go file whose dolt image pin they compare. go_test( name = "scripts_test", srcs = [ @@ -21,14 +40,7 @@ go_test( "bazel_hermetic_cc_test.go", "bazel_policy_test.go", "bazel_rbe_prewarm_test.go", - "bazel_skip_allowlist_test.go", - "check_build_tags_test.go", - "check_doc_freshness_test.go", - "check_release_cross_compile_test.go", - "check_testing_short_test.go", - "check_winget_portable_alias_test.go", "ci_actions_policy_test.go", - "ci_analytics_extract_test.go", "ci_analytics_workflow_test.go", "ci_blacksmith_runner_test.go", "ci_f7b_review_test.go", @@ -36,40 +48,24 @@ go_test( "ci_merge_queue_test.go", "ci_workflow_apt_get_timeout_test.go", "ci_workflow_test.go", - "complexity_script_test.go", - "conformance_script_test.go", "embedded_shard_count_test.go", "host_tools_test.go", - "install_dolt_test.go", "makefile_bazel_primary_test.go", - "managed_local_lane_test.go", "migration_corpus_bazel_test.go", "nogo_lint_policy_test.go", "pr_lanes_bazel_coverage_test.go", - "pr_preflight_test.go", "pr_risk_bazel_coverage_test.go", - "pre_push_suite_test.go", - "pull_dolt_image_test.go", "rbe_worker_env_test.go", - "release_script_test.go", "repo_helpers_test.go", - "required_suite_test.go", - "test_script_test.go", - "upgrade_smoke_loop_test.go", - ], + ], # keep data = [ - "//:repo_files", - "@beads_formulas//:beads-release.formula.toml", + "//:repo_other_files", + "//internal/testutil:testdoltcommon.go", "@go_sdk//:bin/go", "@go_sdk//:files", ], - # The release formula lives in .beads/ (in .bazelignore: it also holds - # live bd data), so it comes in through its own local repository. The Go - # SDK is go.mod's toolchain, whatever go the executor has on PATH. - env = { - "BEADS_TEST_GO": "$(rlocationpath @go_sdk//:bin/go)", - "BEADS_TEST_RELEASE_FORMULA": "$(rlocationpath @beads_formulas//:beads-release.formula.toml)", - }, + # The Go SDK is go.mod's toolchain, whatever go the executor has on PATH. + env = {"BEADS_TEST_GO": "$(rlocationpath @go_sdk//:bin/go)"}, deps = [ "//internal/formula", "//internal/testutil/bazeltest", @@ -78,13 +74,129 @@ go_test( ], ) +# The shell scripts' own behavior, run against fixtures: this package's +# scripts and their testdata, the few root files they read, and the release +# formula. +go_test( + name = "shell_scripts_test", + srcs = [ + "check_build_tags_test.go", + "check_doc_freshness_test.go", + "check_release_cross_compile_test.go", + "check_testing_short_test.go", + "check_winget_portable_alias_test.go", + "ci_analytics_extract_test.go", + "complexity_script_test.go", + "conformance_script_test.go", + "host_tools_test.go", + "install_dolt_test.go", + "pr_preflight_test.go", + "pre_push_suite_test.go", + "pull_dolt_image_test.go", + "release_script_test.go", + "repo_helpers_test.go", + "required_suite_test.go", + "test_script_test.go", + "upgrade_smoke_loop_test.go", + ], + data = [ + ":repo_other_files", + "//:.buildflags", + "//:.githooks/pre-push", + "//:.test-skip", + "//internal/httpclient:repo_other_files", + "//test/conformance:repo_go_test_srcs", + "//test/conformance:repo_other_files", + "//tools/bazel:repo_other_files", + "@beads_formulas//:beads-release.formula.toml", + ], + # The release formula lives in .beads/ (in .bazelignore: it also holds + # live bd data), so it comes in through its own local repository. + env = {"BEADS_TEST_RELEASE_FORMULA": "$(rlocationpath @beads_formulas//:beads-release.formula.toml)"}, + deps = [ + "//internal/formula", + "//internal/testutil/bazeltest", + "//scripts/internal/testbash", + ], +) + +# Scans of the Go test files: the Bazel-skip allowlist, the managed local +# lane, and the shard manifests and scripts against the tests they shard. +# Every _test.go file (//:repo_go_test_srcs) but no non-test Go source, so a +# non-test Go change leaves it cached. +go_test( + name = "go_test_sources_test", + srcs = [ + "bazel_skip_allowlist_test.go", + "embedded_shard_count_test.go", + "host_tools_test.go", + "managed_local_lane_test.go", + "repo_helpers_test.go", + "scripts_targets_test.go", + "shard_manifest_test.go", + ], + data = [ + "BUILD.bazel", + "ci/_embedded_shard_manifest_lib.py", + "ci/embedded_cmd_test_durations.json", + "ci/embedded_storage_test_durations.json", + "ci/gen_embedded_cmd_shard_manifest.py", + "ci/gen_embedded_storage_shard_manifest.py", + "ci/gen_proxied_shard_manifest.py", + "ci/proxied_test_durations.json", + "//:.github/scripts/embedded-cmd-test-shards.txt", + "//:.github/scripts/embedded-storage-test-shard.sh", + "//:.github/scripts/embedded-storage-test-shards.txt", + "//:.github/scripts/embedded-test-shard.sh", + "//:.github/scripts/proxied-cmd-test-shards.txt", + "//:.github/scripts/proxied-test-shard.sh", + "//:.github/scripts/server-storage-test-shard.sh", + "//:.github/scripts/server-storage-test-shards.txt", + "//:nested_go_modules", + "//:repo_go_test_srcs", + "//cmd/bd:repo_other_files", + "//internal/storage/dolt:repo_other_files", + "//internal/storage/embeddeddolt:repo_other_files", + "//tools/bazel:check_shard_coverage.py", + "//tools/bazel:equivalence_allowlist.txt", + ], + deps = ["//internal/testutil/bazeltest"], +) + +# check-testing-short.sh over every Go file, test and non-test. +go_test( + name = "go_sources_test", + srcs = [ + "repo_helpers_test.go", + "testing_short_tree_test.go", + ], + data = [ + "check-testing-short.sh", + "//:repo_go_srcs", + "//:repo_go_test_srcs", + ], + deps = ["//internal/testutil/bazeltest"], +) + +# Scans of every tracked file (//:repo_files): any change re-runs them, so +# keep them few and fast. +go_test( + name = "tracked_files_test", + srcs = [ + "repo_helpers_test.go", + "tracked_files_test.go", + ], + data = ["//:repo_files"], + deps = ["//internal/testutil/bazeltest"], +) + # pr.yml's check-doc-freshness-platforms Linux leg: `go test # -tags integration ./scripts -count=1 -run '^(TestDocFreshness.*| # TestRequiredSuiteContract)$' -args -required-suite=doc-freshness`, whose # TestMain fails unless exactly the doc-freshness inventory is compiled and -# selected. scripts_test runs those tests too (in the integration lane); this -# target adds the inventory and selection contract, over only the files the -# suite reads (its fixtures copy check-doc-freshness.sh). The macOS and +# selected. shell_scripts_test runs those tests too (in the integration lane); +# this target adds the inventory and selection contract, over only the files +# the suite reads (its fixtures copy check-doc-freshness.sh). The macOS and # Windows legs stay in pr.yml. # Tags: # integration-only: every TestDocFreshness test is //go:build integration, @@ -154,15 +266,66 @@ exports_files( visibility = ["//scripts/check-versions:__pkg__"], ) +# Scripts the docs and agent instructions link to by path, which +# //test/docsync checks resolve. +exports_files( + [ + "bump-version.sh", + "check-cli-docs-drift.sh", + "generate-cli-docs.sh", + "test.sh", + ], + visibility = ["//test/docsync:__pkg__"], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -171,7 +334,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/README.md b/scripts/README.md index b74ed4a2e..d6c073ba6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -38,7 +38,7 @@ The broad Go wrappers also cap package and test parallelism to `4` by default shared hosts into a different test topology than GitHub Actions. `scripts/check-testing-short.sh` (`make check-testing-short`; CI runs it in -`//scripts:scripts_test`) enforces +`//scripts:go_sources_test`) enforces that `testing.Short()` is only used for runtime, stress, or large-fixture skips. Use build tags, environment checks, or named wrappers for integration/e2e/API boundaries. diff --git a/scripts/bazel_autofix_push_test.go b/scripts/bazel_autofix_push_test.go index ced5b04b9..9ced90c7d 100644 --- a/scripts/bazel_autofix_push_test.go +++ b/scripts/bazel_autofix_push_test.go @@ -161,17 +161,6 @@ func writeRawPatch(t *testing.T, body string) string { return file } -// requireAutofixBash skips when bash cannot run the autofix scripts: they run -// on Linux runners and use bash 4 associative arrays, and macOS runners ship -// /bin/bash 3.2. -func requireAutofixBash(t *testing.T) { - t.Helper() - requireHostTool(t, "bash") - if err := exec.Command("bash", "-c", "declare -A probe=()").Run(); err != nil { - t.Skip("bash lacks associative arrays (bash >= 4 required)") - } -} - func runAutofixCheck(t *testing.T, script, dir, patch string) (string, error) { t.Helper() requireAutofixBash(t) diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index 2ea1441cc..cf5f1e31f 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -1,8 +1,6 @@ package scripts_test import ( - "bufio" - "bytes" "errors" "os" "os/exec" @@ -11,8 +9,6 @@ import ( "strconv" "strings" "testing" - - "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // Policy tests for the side-by-side Bazel configuration. They are plain Go @@ -20,31 +16,6 @@ import ( // alike. Each invariant is a pure check function exercised against the real // repository files and against synthetic fixtures that break it. -// bazelPolicyRoot returns the repository root holding the Bazel policy files. -// Under `bazel test` those files are declared as data (//:bazel_policy_files) -// and resolved from the runfiles tree; under `go test` the source checkout is -// used directly. -func bazelPolicyRoot(t *testing.T) string { - t.Helper() - if srcdir := os.Getenv("TEST_SRCDIR"); srcdir != "" { - workspace := os.Getenv("TEST_WORKSPACE") - if workspace == "" { - workspace = "_main" - } - return filepath.Join(srcdir, workspace) - } - return sourceRepoRoot(t) -} - -func readPolicyFile(t *testing.T, root, name string) string { - t.Helper() - data, err := os.ReadFile(filepath.Join(root, name)) - if err != nil { - t.Fatalf("read %s: %v", name, err) - } - return string(data) -} - // --- go_sdk version == go.mod toolchain ------------------------------------- var ( @@ -304,366 +275,11 @@ func TestBazelLocalRCFilesGitignored(t *testing.T) { } } -// --- no remote-execution endpoints in tracked files ------------------------- - -// Remote-execution endpoints, credentials and TLS material belong in a -// gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the -// workspace, never in the repository. The scan covers committed bazelrc-like -// files, Markdown docs and .github/** (plan §3.8) and flags: -// - any remote/BES/TLS flag below whose value is a literal (not a -// , a $VARIABLE/${{ expression }}, empty, or loopback); -// - in rc and .github files, any gRPC URL that is not loopback. Markdown is -// exempt from this rule because beads documents OTel gRPC exporters. -// -// The one exception is publicForkCacheLines. - -var ( - remoteFlagNames = `remote_executor|remote_cache|remote_downloader|remote_header|remote_instance_name|` + - `bes_backend|bes_results_url|bes_header|tls_client_certificate|tls_client_key|tls_certificate` - // `--flag=value` is recognized everywhere; `--flag value` only in rc and - // workflow files, where prose ("--remote_executor and ...") does not occur. - remoteFlagEqRe = regexp.MustCompile(`--(` + remoteFlagNames + `)=("[^"]*"|'[^']*'|[^\s"'` + "`" + `]*)`) - remoteFlagSpaceRe = regexp.MustCompile(`--(` + remoteFlagNames + `)\s+("[^"]*"|'[^']*'|[^\s"'` + "`" + `]+)`) - remoteEndpointRe = regexp.MustCompile(`\bgrpcs?://[^\s"'<>)\]]+`) - loopbackValueRe = regexp.MustCompile(`^(?:[a-z][a-z0-9+.-]*://)?(?:127\.0\.0\.1|localhost|\[::1\])(?:[:/]|$)`) - // remoteScanPrefilter: a file lacking all of these cannot produce a hit. - remoteScanPrefilter = [][]byte{[]byte("--remote_"), []byte("--bes_"), []byte("--tls_"), []byte("grpc")} -) - -// publicForkCacheLines: the endpoint lines of .bazelrc's fork-cache config, -// public by design. rbe-west's rbe-cache endpoint is anonymous and read-only -// (action-cache and CAS reads; writes and Execute are refused by the farm), -// so there is no credential to leak. Exactly these lines, trimmed, and only -// in .bazelrc; the same line anywhere else, or any other endpoint, is still a -// hit. Assembled so this file never contains a literal endpoint. -var publicForkCacheLines = map[string]bool{ - "build:fork-cache --remote_cache=" + forkCacheEndpoint: true, - "build:fork-cache --remote_instance_name=" + forkCacheInstance: true, -} - -const ( - forkCacheEndpoint = "grpc" + "s://rbe-cache.ops.gascity.com:8443" - forkCacheInstance = "oss" - // zstd cache transfers: only the anonymous fork cache can advertise a - // compressor, so only fork-cache may ask for one, and only while - // write-bazelrc.sh's cache-zstd-probe.sh finds it advertised (a probe, - // not a repository variable: fork pull_request runs see no vars). - // Trusted remote-exec and rbe-fork never: their schedulers advertise - // none, and Bazel then refuses the remote. - forkCacheZstdLine = "build:fork-cache --remote_cache_compression" -) - -// publicRBEForkPin: setup-bazel's fork-credential.sh pins the one endpoint -// rbe-fork-mint may hand a fork run (rbe-fork, :8444), so a compromised -// mint cannot point fork builds elsewhere. It is public and carries no -// credential: allowed as exactly this line, in exactly that file (a byte -// copy of gascity's tools/rbe/fork-credential.sh). -const ( - publicRBEForkPinFile = ".github/actions/setup-bazel/fork-credential.sh" - publicRBEForkPinLine = `ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-fork\.ops\.gascity\.com:8444$'}` -) - -type endpointHit struct { - path string - line int - what string -} - -// remoteScanKind classifies a tracked path for the endpoint scan. -func remoteScanKind(rel string) (scan, strict bool) { - base := filepath.Base(rel) - switch { - case strings.Contains(base, "bazelrc"), strings.HasPrefix(rel, ".github/"): - return true, true - case strings.HasSuffix(base, ".md"): - return true, false - } - return false, false -} - -func allowedRemoteValue(v string) bool { - v = strings.Trim(v, `"'`) - if i := strings.Index(v, "://"); i >= 0 { - v = v[i+len("://"):] - } - return v == "" || strings.HasPrefix(v, "<") || strings.HasPrefix(v, "$") || loopbackValueRe.MatchString(v) -} - -// findRemoteEndpoints reports literal remote endpoints/credentials in content. -// strict enables the whitespace-separated flag form and the gRPC URL rule. -func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit { - if bytes.IndexByte(content, 0) >= 0 { // binary - return nil - } - relevant := false - for _, needle := range remoteScanPrefilter { - if bytes.Contains(content, needle) { - relevant = true - break - } - } - if !relevant { - return nil - } - flagRes := []*regexp.Regexp{remoteFlagEqRe} - if strict { - flagRes = append(flagRes, remoteFlagSpaceRe) - } - var hits []endpointHit - scanner := bufio.NewScanner(bytes.NewReader(content)) - scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) - for n := 1; scanner.Scan(); n++ { - line := scanner.Text() - if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] { - continue - } - if path == publicRBEForkPinFile && line == publicRBEForkPinLine { - continue - } - for _, re := range flagRes { - for _, m := range re.FindAllStringSubmatch(line, -1) { - if !allowedRemoteValue(m[2]) { - hits = append(hits, endpointHit{path: path, line: n, what: m[0]}) - } - } - } - if strict { - for _, url := range remoteEndpointRe.FindAllString(line, -1) { - if !allowedRemoteValue(url) { - hits = append(hits, endpointHit{path: path, line: n, what: url}) - } - } - } - } - return hits -} - -func gitRepoAvailable(root string) bool { - if _, err := exec.LookPath("git"); err != nil { - return false - } - return exec.Command("git", "-C", root, "rev-parse", "--is-inside-work-tree").Run() == nil -} - -// repoFiles lists the repository's tracked files under root, repo-relative and -// slash-separated, each a regular file (or, in a local runfiles tree, a -// symlink to one). Under `go test` in a git checkout that is `git ls-files` -// less what the working tree no longer holds; under Bazel it is the runfiles -// tree, whose repository part is //:repo_files (every tracked file outside -// .bazelignore, tools/bazel/go_srcs.py); elsewhere every file under root. -func repoFiles(t *testing.T, root string) []string { - t.Helper() - var files []string - if bazeltest.IsBazel() || !gitRepoAvailable(root) { - err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !isFileOrFileLink(path, d) { - return nil - } - rel, err := filepath.Rel(root, path) - if err != nil { - return err - } - files = append(files, filepath.ToSlash(rel)) - return nil - }) - if err != nil { - t.Fatalf("walk %s: %v", root, err) - } - } else { - out, err := exec.Command("git", "-C", root, "ls-files", "-z").Output() - if err != nil { - t.Fatalf("git ls-files: %v", err) - } - for _, rel := range strings.Split(strings.TrimRight(string(out), "\x00"), "\x00") { - info, err := os.Lstat(filepath.Join(root, filepath.FromSlash(rel))) - if err != nil || !info.Mode().IsRegular() { - continue // deleted in the worktree, submodule, or symlink - } - files = append(files, rel) - } - } - // Sanity: an empty or partial runfiles tree would pass every scan. - if len(files) < 1000 { - t.Fatalf("found only %d repository files under %s; the listing is broken", len(files), root) - } - return files -} - -// isFileOrFileLink reports whether a WalkDir entry is a regular file or a -// symlink to one. Bazel's local runfiles trees are symlink forests, so a walk -// that skipped symlinks would see no file there; a symlink to a directory -// (Bazel's bazel-* convenience links in a checkout) is never followed. -func isFileOrFileLink(path string, d os.DirEntry) bool { - if d.Type().IsRegular() { - return true - } - if d.Type()&os.ModeSymlink == 0 { - return false - } - info, err := os.Stat(path) - return err == nil && info.Mode().IsRegular() -} - -func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) { - // Fixtures are assembled at runtime so this file never contains a literal - // endpoint itself. - scheme := "grpc" + "s://" - flag := func(name string) string { return "--" + name } - for _, bad := range []string{ - "build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe.example.com:443\n", - "build:remote-exec " + flag("remote_executor") + "=rbe.example.com:443\n", - "build:remote-exec " + flag("remote_executor") + " rbe.example.com:443\n", - "build " + flag("remote_cache") + "=https://cache.example.com\n", - "build " + flag("bes_backend") + "=bes.example.com\n", - "build " + flag("bes_results_url") + "=https://results.example.com/inv/\n", - "build " + flag("remote_header") + "=x-api-key=abc123\n", - "build " + flag("tls_client_certificate") + "=/etc/rbe/client.crt\n", - "build " + flag("tls_client_key") + "=\"/etc/rbe/client.key\"\n", - "# farm: " + scheme + "10.0.0.5:8980\n", - "REMOTE=" + "grpc" + "://cache.internal:9092\n", - } { - if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 { - t.Errorf("endpoint fixture not detected: %q", bad) - } - } - for _, ok := range []string{ - "build:remote-exec " + flag("remote_executor") + "=" + scheme + "\n", - "build:remote-exec " + flag("remote_executor") + "=\n", - "build:remote-exec " + flag("tls_client_certificate") + "=\n", - " run: echo \"build " + flag("remote_executor") + "=${{ secrets.BAZEL_REMOTE_EXECUTOR }}\" >> \"$RUNNER_TEMP/ci.bazelrc\"\n", - " " + flag("tls_client_key") + " \"$RUNNER_TEMP/rbe.key\"\n", - " " + flag("remote_executor") + "=" + scheme + "${{ secrets.RBE_HOST }}\n", - flag("remote_cache") + "=" + "grpc" + "://127.0.0.1:50052\n", - flag("remote_cache") + "=localhost:9092\n", - flag("remote_cache") + "=\n", - "build " + flag("remote_timeout") + "=600\n", - "use grpc for transport\n", - } { - if hits := findRemoteEndpoints(".bazelrc", []byte(ok), true); len(hits) != 0 { - t.Errorf("allowed fixture flagged: %q -> %v", ok, hits) - } - } - // The fork cache's public endpoint: exactly its two .bazelrc lines. - forkCache := "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n" + - " build:fork-cache " + flag("remote_instance_name") + "=oss \n" - if hits := findRemoteEndpoints(".bazelrc", []byte(forkCache), true); len(hits) != 0 { - t.Errorf("fork-cache endpoint lines flagged in .bazelrc: %v", hits) - } - for _, path := range []string{"tools/ci.bazelrc", ".bazelrc.local", "user.bazelrc", ".github/workflows/x.yml", ".github/actions/setup-bazel/x.sh"} { - if hits := findRemoteEndpoints(path, []byte(forkCache), true); len(hits) < 2 { - t.Errorf("fork-cache endpoint lines in %s: hits %v, want both lines flagged (allowlisted in .bazelrc only)", path, hits) - } - } - for _, bad := range []string{ - "build:fork-cache " + flag("remote_cache") + "=" + scheme + "other.example:8443\n", - "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "/x\n", - "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + " " + flag("remote_header") + "=x-api-key=abc\n", - "build " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n", - "build:remote-exec " + flag("remote_executor") + "=" + forkCacheEndpoint + "\n", - "build:fork-cache " + flag("remote_instance_name") + "=beads\n", - "# see " + forkCacheEndpoint + "\n", - } { - if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 { - t.Errorf("non-allowlisted fork-cache variant not detected in .bazelrc: %q", bad) - } - } - // rbe-fork's endpoint pin: that line in that file only; anything else - // naming the endpoint, there or elsewhere, is flagged. - if hits := findRemoteEndpoints(publicRBEForkPinFile, []byte("set -eu\n"+publicRBEForkPinLine+"\n"), true); len(hits) != 0 { - t.Errorf("rbe-fork endpoint pin flagged in %s: %v", publicRBEForkPinFile, hits) - } - for _, path := range []string{".github/actions/setup-bazel/write-bazelrc.sh", ".github/workflows/bazel.yml", ".bazelrc", "tools/rbe/fork-credential.sh"} { - if len(findRemoteEndpoints(path, []byte(publicRBEForkPinLine+"\n"), true)) == 0 { - t.Errorf("rbe-fork endpoint pin not flagged in %s (allowlisted in %s only)", path, publicRBEForkPinFile) - } - } - for _, bad := range []string{ - " " + publicRBEForkPinLine, - strings.Replace(publicRBEForkPinLine, "8444", "443", 1), - `ENDPOINT=` + "grpc" + `s://rbe-fork.ops.gascity.com:8444`, - "build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe-fork.ops.gascity.com:8444", - } { - if len(findRemoteEndpoints(publicRBEForkPinFile, []byte(bad+"\n"), true)) == 0 { - t.Errorf("non-allowlisted rbe-fork line not detected in %s: %q", publicRBEForkPinFile, bad) - } - } - // Markdown: OTel gRPC exporter URLs and prose mentioning the flag are fine; - // a literal flag value is not. - for _, ok := range []string{ - "Set OTEL_EXPORTER_OTLP_ENDPOINT=" + "grpc" + "://collector:4317\n", - "Pass " + flag("remote_executor") + " and TLS flags from user.bazelrc.\n", - } { - if hits := findRemoteEndpoints("docs/x.md", []byte(ok), false); len(hits) != 0 { - t.Errorf("allowed Markdown fixture flagged: %q -> %v", ok, hits) - } - } - if len(findRemoteEndpoints("docs/x.md", []byte("bazel build "+flag("remote_executor")+"=rbe.example.com:443 //...\n"), false)) == 0 { - t.Error("literal remote_executor in Markdown not detected") - } - for rel, want := range map[string][2]bool{ - ".bazelrc": {true, true}, - "tools/ci.bazelrc": {true, true}, - ".bazelrc.local": {true, true}, - ".github/workflows/x.yml": {true, true}, - "docs/BAZEL.md": {true, false}, - "issues.jsonl": {false, false}, - "internal/telemetry/x.go": {false, false}, - } { - if scan, strict := remoteScanKind(rel); scan != want[0] || strict != want[1] { - t.Errorf("remoteScanKind(%q) = %v,%v, want %v,%v", rel, scan, strict, want[0], want[1]) - } - } - - root := bazelPolicyRoot(t) - var hits []endpointHit - for _, rel := range repoFiles(t, root) { - scan, strict := remoteScanKind(rel) - if !scan { - continue - } - content, err := os.ReadFile(filepath.Join(root, rel)) - if err != nil { - t.Fatalf("read %s: %v", rel, err) - } - hits = append(hits, findRemoteEndpoints(rel, content, strict)...) - } - for _, h := range hits { - t.Errorf("%s:%d: remote endpoint or credential %q in a tracked file; it belongs in a gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the workspace", h.path, h.line, h.what) - } -} - -// TestNoLocalPlanPathsInTrackedFiles: tracked files must not point readers -// at a maintainer's private, out-of-repo planning notes (a home-directory -// planning-notes tree), which no other contributor can open. Cite an -// in-repo doc, a bead, or a PR instead. The needle is assembled at runtime so -// this file does not match itself. -func TestNoLocalPlanPathsInTrackedFiles(t *testing.T) { - root := bazelPolicyRoot(t) - needle := []byte("beads-" + "bazel-plan") - for _, rel := range repoFiles(t, root) { - content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel))) - if err != nil { - t.Fatalf("read %s: %v", rel, err) - } - if bytes.IndexByte(content, 0) >= 0 { - continue // binary, as git grep -I - } - for n, line := range bytes.Split(content, []byte("\n")) { - if bytes.Contains(line, needle) { - t.Errorf("%s:%d: tracked file references a local, non-repo %s path; point at an in-repo doc, bead or PR instead", rel, n+1, needle) - } - } - } -} - // --- generated go_srcs filegroups are current ------------------------------- -// These checks walk the source checkout: under Bazel, //:repo_files, which -// holds every package's BUILD.bazel (a package missing its repo_files block -// is what `make bazel-sync-check` fails on). +// These checks walk the source checkout: under Bazel, //:repo_other_files, +// which holds every package's BUILD.bazel (a package missing its repo_files +// block is what `make bazel-sync-check` fails on). var ( treeGoSrcsRe = regexp.MustCompile(`(?s)filegroup\(\s*name\s*=\s*"tree_go_srcs",\s*srcs\s*=\s*\[(.*?)\]`) @@ -791,7 +407,7 @@ func TestBazelGoSrcsBlocksCurrent(t *testing.T) { // --- no Bazel packages under the docs trees --------------------------------- // docs/ and engdocs/ are content (the Mintlify site and the engineering docs) -// that the root package's repo_files glob covers; they hold no code, so a +// that the root package's repo_files block covers; they hold no code, so a // BUILD file under either tree is a mistake (it would also split the subtree // into a package of its own). func TestBazelNoPackagesUnderDocsTrees(t *testing.T) { diff --git a/scripts/bazel_skip_allowlist_test.go b/scripts/bazel_skip_allowlist_test.go index b90d6c19f..dd8b7901b 100644 --- a/scripts/bazel_skip_allowlist_test.go +++ b/scripts/bazel_skip_allowlist_test.go @@ -27,7 +27,7 @@ import ( // tests run on every PR, so a go-test-only check (a Bazel-guarded return or // continue, or a block guarded by TEST_SRCDIR == "" / !IsBazel()) would run // nowhere: declare what it reads as data instead (//:repo_files holds the -// whole checkout). This check finds them before the merge, from the source. +// whole checkout, tools/bazel/go_srcs.py's partitions of it less). This check finds them before the merge, from the source. // // Only top-level tests are considered: a subtest's skip is invisible to // equivalence.py, which compares top-level tests. @@ -39,8 +39,8 @@ func TestBazelOnlySkipsAreAllowlisted(t *testing.T) { if err != nil { t.Fatal(err) } - // Under Bazel the tree is //:repo_files; a scan that parsed few files is - // not looking at it. + // Under Bazel the tree is //:repo_go_test_srcs; a scan that parsed few + // files is not looking at it. if files < 1000 { t.Fatalf("parsed only %d _test.go files; the scan is broken", files) } @@ -366,20 +366,6 @@ func checks(body *ast.BlockStmt) bool { return found } -func exits(body *ast.BlockStmt) bool { - for _, st := range body.List { - switch st := st.(type) { - case *ast.ReturnStmt: - return true - case *ast.BranchStmt: - if st.Tok == token.CONTINUE { - return true - } - } - } - return false -} - // goTestCond: TEST_SRCDIR == "" or !...IsBazel(). func goTestCond(cond ast.Expr) bool { switch c := cond.(type) { diff --git a/scripts/bench-ready-indexes/BUILD.bazel b/scripts/bench-ready-indexes/BUILD.bazel index d67926583..185a441a1 100644 --- a/scripts/bench-ready-indexes/BUILD.bazel +++ b/scripts/bench-ready-indexes/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/check-versions/BUILD.bazel b/scripts/check-versions/BUILD.bazel index 6baf8b20d..818f31552 100644 --- a/scripts/check-versions/BUILD.bazel +++ b/scripts/check-versions/BUILD.bazel @@ -42,12 +42,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -56,7 +95,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/check_testing_short_test.go b/scripts/check_testing_short_test.go index 9e1257766..b2881db63 100644 --- a/scripts/check_testing_short_test.go +++ b/scripts/check_testing_short_test.go @@ -66,26 +66,6 @@ func TestCheckTestingShortReportsUnknownBetweenFunctions(t *testing.T) { } } -func TestCheckTestingShortPassesOnCleanRepoTree(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("checker is a Bash boundary") - } - repo := sourceRepoRoot(t) - // Under Bazel the tree is //:repo_files; a tree missing the allowlisted - // call sites would let the scan pass vacuously. - for _, rel := range []string{"internal/hooks/hooks_test.go", "internal/storage/dolt/concurrent_test.go", "internal/workapi/sweep_test.go"} { - if _, err := os.Stat(filepath.Join(repo, filepath.FromSlash(rel))); err != nil { - t.Fatalf("the scanned tree lacks %s, which holds an allowlisted call: %v", rel, err) - } - } - cmd := exec.Command("bash", filepath.Join(repo, "scripts", "check-testing-short.sh")) - cmd.Dir = repo - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("check-testing-short.sh failed on clean repo tree (all 9 allowlist entries must still be recognized): %v\noutput: %s", err, out) - } -} - func runCheckTestingShort(t *testing.T, files map[string]string) (string, error) { t.Helper() if runtime.GOOS == "windows" { diff --git a/scripts/check_winget_portable_alias_test.go b/scripts/check_winget_portable_alias_test.go index 8a52fcc79..0e5a5e56d 100644 --- a/scripts/check_winget_portable_alias_test.go +++ b/scripts/check_winget_portable_alias_test.go @@ -11,7 +11,7 @@ import ( // The guard's own vocabulary is shared between mikefarah yq v4 and python-yq, // so either flavour exercises these cases. Under Bazel yq is part of the -// executor inventory //scripts:scripts_test relies on (the rbe-west worker +// executor inventory the //scripts go_tests rely on (the rbe-west worker // image and ubuntu-latest ship mikefarah yq v4), so a missing yq fails there. func skipWithoutYQ(t *testing.T) { t.Helper() diff --git a/scripts/checkworkflowtags/BUILD.bazel b/scripts/checkworkflowtags/BUILD.bazel index 1a80d8372..2ce173f43 100644 --- a/scripts/checkworkflowtags/BUILD.bazel +++ b/scripts/checkworkflowtags/BUILD.bazel @@ -17,12 +17,51 @@ go_binary( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 6905ece3c..eb75ab320 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2516,7 +2516,7 @@ func TestPRPolicyChecksRunAsBazelTargets(t *testing.T) { {"scripts/repochecks/BUILD.bazel", "workapi_frontend_boundary_test", `"scripts/check-workapi-frontend-boundary.sh"`, ""}, {"scripts/repochecks/BUILD.bazel", "types_gen_drift_test", `"types_gen_drift_test.sh"`, ""}, {"internal/versioncheck/BUILD.bazel", "versioncheck_test", `"//:release_metadata_files"`, ""}, - {"scripts/BUILD.bazel", "scripts_test", `"check_testing_short_test.go"`, ""}, + {"scripts/BUILD.bazel", "go_sources_test", `"testing_short_tree_test.go"`, ""}, {"scripts/BUILD.bazel", "doc_freshness_required_test", `"-required-suite=doc-freshness"`, `["integration-only"]`}, {"scripts/gitattributespolicy/BUILD.bazel", "gitattributespolicy_required_host_test", `"-required-host"`, ""}, } { @@ -3896,9 +3896,6 @@ sh_test( } } -// The default shard manifest of a PR Risk shard script. -var shardManifestDefault = regexp.MustCompile(`\$\{BEADS_TEST_SHARD_MANIFEST:-([^}]+)\}`) - // bazelAttrBlock returns the text of a list attribute (` name = [` up to // its closing ` ],`) of a rule block from bazelRuleBlock, or "". func bazelAttrBlock(rule, name string) string { @@ -3913,40 +3910,6 @@ func bazelAttrBlock(rule, name string) string { return rule[i : i+end+7] } -// bazelRuleBlock returns the text of the top-level rule named name in a -// BUILD file, or "" if there is none. -func bazelRuleBlock(build, name string) string { - i := strings.Index(build, "\n name = \""+name+"\",\n") - if i < 0 { - return "" - } - start := strings.LastIndex(build[:i], "\n") + 1 - end := strings.Index(build[i:], "\n)\n") - if end < 0 { - return build[start:] - } - return build[start : i+end+3] -} - -// bazelProxiedShardCount returns cmd/bd:bd_proxied_test's own shard_count -// from cmd/bd/BUILD.bazel: the single source of truth for the Bazel-only -// bazel-proxied lane's shard split, which no longer has to equal PR -// Risk's/main.yml's legacy test-proxied-cmd jobs' matrix size (F2). -func bazelProxiedShardCount(t *testing.T) int { - t.Helper() - root := sourceRepoRoot(t) - rule := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_proxied_test") - m := regexp.MustCompile(`(?m)^ shard_count = (\d+),$`).FindStringSubmatch(rule) - if m == nil { - t.Fatalf("cmd/bd:bd_proxied_test has no `shard_count = N,` in cmd/bd/BUILD.bazel:\n%s", rule) - } - n, err := strconv.Atoi(m[1]) - if err != nil { - t.Fatalf("cmd/bd:bd_proxied_test shard_count: %v", err) - } - return n -} - // bazel-integration mirrors main.yml's integration jobs with the unmodified // --config=integration (whose .bazelrc filter and flags // TestBazelrcIntegrationLane pins): no extra filter, selector or remote @@ -5755,6 +5718,13 @@ func TestBazelGatedLanesNeverRetryFlakyTests(t *testing.T) { case ".git", "node_modules", ".beads": return filepath.SkipDir } + // Not in the checkout: under Bazel, the --run_under wrapper's + // PATH directory (//tools/bazel:hermetic_bin) is in every + // test's runfiles, and a line-by-line scan of its pinned dolt + // binary took a minute on a remote worker. + if filepath.ToSlash(rel) == "tools/bazel/hermetic_bin" { + return filepath.SkipDir + } return nil } if !isFileOrFileLink(path, d) { @@ -6068,3 +6038,31 @@ func (e *ciWorkflowEnvironment) UnmarshalYAML(value *yaml.Node) error { e.Name, e.URL = m.Name, m.URL return nil } + +func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { + // No workflow pulls the image: every Dolt suite runs on hermetic dolt + // sql-servers (bazel.yml's dolt-server lanes) and needs no docker, since + // pr-risk.yml's container-backed legacy tiers were retired + // (ga-96smfk.22). The helper stays for local container-backed runs. + wantCalls := map[string]int{} + + workflowsDir := filepath.Join(sourceRepoRoot(t), ".github", "workflows") + workflowPaths, err := filepath.Glob(filepath.Join(workflowsDir, "*.y*ml")) + if err != nil { + t.Fatal(err) + } + for _, path := range workflowPaths { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + text := string(data) + name := filepath.Base(path) + if got, want := strings.Count(text, "run: ./scripts/ci/pull-dolt-image.sh"), wantCalls[name]; got != want { + t.Errorf("%s retry-helper calls = %d, want %d", name, got, want) + } + if strings.Contains(text, "docker pull "+doltSQLServerImage) { + t.Errorf("%s still pulls the Dolt image without retries", name) + } + } +} diff --git a/scripts/embedded_shard_count_test.go b/scripts/embedded_shard_count_test.go index dcc75521f..f439235be 100644 --- a/scripts/embedded_shard_count_test.go +++ b/scripts/embedded_shard_count_test.go @@ -1,14 +1,18 @@ package scripts_test +// The Bazel shard counts the CI policy tests (//scripts:scripts_test) and the +// shard-manifest tests (//scripts:go_test_sources_test) check against, read +// from each sharded target's own BUILD.bazel rule. Helpers only: both targets +// compile this file, so a test here would run twice. + import ( - "os/exec" "regexp" "strconv" "testing" ) // shardCountPattern matches a rule block's ` shard_count = N,` line, the -// same pattern bazelProxiedShardCount (scripts/ci_workflow_test.go) inlines +// same pattern bazelProxiedShardCount (below) inlines // for its own single use; shared here since both embedded accessors below // need it. var shardCountPattern = regexp.MustCompile(`(?m)^ shard_count = (\d+),$`) @@ -18,7 +22,7 @@ var shardCountPattern = regexp.MustCompile(`(?m)^ shard_count = (\d+),$`) // Bazel-only bazel-embedded lane's cmd/bd shard split (slice F1), which no // longer has to equal PR Risk's/main.yml's legacy "Test (Embedded Dolt Cmd // N/20)" fork/push jobs' matrix size — mirrors bazelProxiedShardCount (F2; -// see that function's doc comment in scripts/ci_workflow_test.go for the +// see that function's doc comment below for the // shared rationale, not repeated here). func bazelEmbeddedCmdShardCount(t *testing.T) int { t.Helper() @@ -57,46 +61,6 @@ func bazelEmbeddedStorageShardCount(t *testing.T) int { return n } -// S3 (F1, mirroring F2's TestProxiedShardManifestGeneratorNotStale in -// scripts/pr_risk_bazel_coverage_test.go — see that test's doc comment for -// the full --check rationale, not repeated here): the Bazel-only 50-shard -// cmd block and 15-shard storage block are not frozen like their files' -// legacy 20- and 5-shard blocks. gen_embedded_{cmd,storage}_shard_manifest.py -// --check verifies only that the committed block names every discovered -// test exactly once, failing with the exact command to fix it when a name -// is missing, stale, or duplicated — run here so a drifted block fails `go -// test ./scripts/...` (//scripts:scripts_test under Bazel) instead of only -// surfacing as a test silently never running in any shard. The legacy -// blocks are deliberately excluded: both files document that their 20- and -// 5-shard blocks are frozen (see .github/scripts/embedded-{cmd,storage}- -// test-shards.txt and engdocs/TESTING.md), so a --check against them is -// expected to report "missing" entries by design (see those generators' -// module docstrings) and is not what this test runs. -func TestCmdEmbeddedShardManifestGeneratorNotStale(t *testing.T) { - python := requireHostTool(t, "python3") - root := sourceRepoRoot(t) - shards := strconv.Itoa(bazelEmbeddedCmdShardCount(t)) - cmd := exec.Command(python, "scripts/ci/gen_embedded_cmd_shard_manifest.py", shards, "--weights=duration", "--check") - cmd.Dir = root - if out, err := cmd.CombinedOutput(); err != nil { - t.Errorf("gen_embedded_cmd_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) - } -} - -// TestStorageEmbeddedShardManifestGeneratorNotStale mirrors -// TestCmdEmbeddedShardManifestGeneratorNotStale above for the storage tier's -// Bazel-only 15-shard block; see that test's doc comment. -func TestStorageEmbeddedShardManifestGeneratorNotStale(t *testing.T) { - python := requireHostTool(t, "python3") - root := sourceRepoRoot(t) - shards := strconv.Itoa(bazelEmbeddedStorageShardCount(t)) - cmd := exec.Command(python, "scripts/ci/gen_embedded_storage_shard_manifest.py", shards, "--weights=duration", "--check") - cmd.Dir = root - if out, err := cmd.CombinedOutput(); err != nil { - t.Errorf("gen_embedded_storage_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) - } -} - // bazelServerFullShardCount returns dolt:dolt_server_full_test's own // shard_count from internal/storage/dolt/BUILD.bazel: the single source of // truth for the bazel-server-storage lane's full-suite shard split, now that @@ -117,3 +81,25 @@ func bazelServerFullShardCount(t *testing.T) int { } return n } + +// The default shard manifest of a PR Risk shard script. +var shardManifestDefault = regexp.MustCompile(`\$\{BEADS_TEST_SHARD_MANIFEST:-([^}]+)\}`) + +// bazelProxiedShardCount returns cmd/bd:bd_proxied_test's own shard_count +// from cmd/bd/BUILD.bazel: the single source of truth for the Bazel-only +// bazel-proxied lane's shard split, which no longer has to equal PR +// Risk's/main.yml's legacy test-proxied-cmd jobs' matrix size (F2). +func bazelProxiedShardCount(t *testing.T) int { + t.Helper() + root := sourceRepoRoot(t) + rule := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_proxied_test") + m := regexp.MustCompile(`(?m)^ shard_count = (\d+),$`).FindStringSubmatch(rule) + if m == nil { + t.Fatalf("cmd/bd:bd_proxied_test has no `shard_count = N,` in cmd/bd/BUILD.bazel:\n%s", rule) + } + n, err := strconv.Atoi(m[1]) + if err != nil { + t.Fatalf("cmd/bd:bd_proxied_test shard_count: %v", err) + } + return n +} diff --git a/scripts/gitattributespolicy/BUILD.bazel b/scripts/gitattributespolicy/BUILD.bazel index c41371b52..d3ced62de 100644 --- a/scripts/gitattributespolicy/BUILD.bazel +++ b/scripts/gitattributespolicy/BUILD.bazel @@ -33,12 +33,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -47,7 +86,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/host_tools_test.go b/scripts/host_tools_test.go index 0c498194b..6773d7d11 100644 --- a/scripts/host_tools_test.go +++ b/scripts/host_tools_test.go @@ -8,8 +8,8 @@ import ( ) // skipOrFailWithoutHostTool handles a missing host tool. Under go test the -// test skips, as it always has. Under Bazel it fails: //scripts:scripts_test -// relies on the executor's bash, git, python3 and friends (the rbe-west +// test skips, as it always has. Under Bazel it fails: the //scripts go_tests +// rely on the executor's bash, git, python3 and friends (the rbe-west // worker image, or the runner in fork-cache mode), so a skip there would // hide an executor that lacks one behind a green, cacheable result. func skipOrFailWithoutHostTool(t *testing.T, format string, args ...any) { @@ -45,3 +45,14 @@ func testGo(t *testing.T) string { } return requireHostTool(t, "go") } + +// requireAutofixBash skips when bash cannot run the autofix scripts: they run +// on Linux runners and use bash 4 associative arrays, and macOS runners ship +// /bin/bash 3.2. +func requireAutofixBash(t *testing.T) { + t.Helper() + requireHostTool(t, "bash") + if err := exec.Command("bash", "-c", "declare -A probe=()").Run(); err != nil { + t.Skip("bash lacks associative arrays (bash >= 4 required)") + } +} diff --git a/scripts/internal/testbash/BUILD.bazel b/scripts/internal/testbash/BUILD.bazel index 6ec7c261e..3c200eee7 100644 --- a/scripts/internal/testbash/BUILD.bazel +++ b/scripts/internal/testbash/BUILD.bazel @@ -29,12 +29,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/migration-test/catalog/BUILD.bazel b/scripts/migration-test/catalog/BUILD.bazel index 70e66d75a..b69d11468 100644 --- a/scripts/migration-test/catalog/BUILD.bazel +++ b/scripts/migration-test/catalog/BUILD.bazel @@ -23,12 +23,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/pr-lint/BUILD.bazel b/scripts/pr-lint/BUILD.bazel index a232d5557..4d30153de 100644 --- a/scripts/pr-lint/BUILD.bazel +++ b/scripts/pr-lint/BUILD.bazel @@ -22,12 +22,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -36,7 +75,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index b2058509f..b53fdfb69 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -416,8 +416,8 @@ func TestDoltServerFingerprintRunsOnTheDoltServerLane(t *testing.T) { // cmd/go's defaultVetFlags analyzers beside every compile of every lane, so // bazel.yml's required test lane (`bazel test //... --config=ci`) fails on a // finding. No workflow runs `go vet` any more. The repository policy tests -// and the Go tests that walk the checkout run under Bazel -// (//scripts:scripts_test and //test/docsync over //:repo_files), so no +// and the Go tests that walk the checkout run under Bazel (//scripts' and +// //test/docsync's go_tests, over the repo_files partitions they read), so no // workflow runs `go test` over ./scripts/... or the equivalence allowlist // either. func TestPRRunsGoTestsBazelSkips(t *testing.T) { diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index edf17a537..b058e806d 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -10,7 +10,6 @@ import ( "sort" "strconv" "strings" - "sync" "testing" ) @@ -319,7 +318,7 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { // liveShardCount dispatches a suite's -1 shardCount to the accessor that // reads its target's own shard_count from its BUILD.bazel rule — the // single source of truth for a lane's shard split. See - // bazelProxiedShardCount's doc comment (scripts/ci_workflow_test.go) for + // bazelProxiedShardCount's doc comment (scripts/embedded_shard_count_test.go) for // the shared rationale. liveShardCount := map[string]func(*testing.T) int{ "//cmd/bd:bd_proxied_test": bazelProxiedShardCount, @@ -553,199 +552,6 @@ esac } } -// Review G1: the checker's input is the real shard scripts' list-only -// output (every retired lane's: the embedded, proxied and server suites). Every name they list, for every shard, must be a test go test -// runs (declared `func Name(t *testing.T)` in the package's _test.go files), -// or one check_shard_coverage.py drops as NOT_TESTS; and each NOT_TESTS name -// must really not be a test (TestMain takes *testing.M). Otherwise the -// checker reports a listed test that "did not run" on every real run. -func TestShardScriptsListOnlyRealTests(t *testing.T) { - // The CI shard scripts use bash 4 associative arrays (Linux runners only). - requireAutofixBash(t) - root := sourceRepoRoot(t) - m := regexp.MustCompile(`(?m)^NOT_TESTS = frozenset\(\{([^}]*)\}\)`).FindStringSubmatch(readPolicyFile(t, root, "tools/bazel/check_shard_coverage.py")) - if m == nil { - t.Fatal("tools/bazel/check_shard_coverage.py has no NOT_TESTS = frozenset({...})") - } - notTests := map[string]bool{} - for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(m[1], -1) { - notTests[q[1]] = true - } - for _, c := range []struct{ job, script, pkg string }{ - {"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd"}, - {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt"}, - // D2 step 2: the proxied and server lanes' suites. - {"test-proxied-cmd", ".github/scripts/proxied-test-shard.sh", "cmd/bd"}, - {"test-server-storage-full", ".github/scripts/server-storage-test-shard.sh", "internal/storage/dolt"}, - } { - var src strings.Builder - files, err := filepath.Glob(filepath.Join(root, c.pkg, "*_test.go")) - if err != nil || len(files) == 0 { - t.Fatalf("%s: no _test.go files (%v)", c.pkg, err) - } - for _, f := range files { - data, err := os.ReadFile(f) - if err != nil { - t.Fatal(err) - } - src.Write(data) - src.WriteString("\n") - } - declared := map[string]bool{} - for _, d := range regexp.MustCompile(`(?m)^func (Test\w*)\(\w+ \*testing\.T\) \{`).FindAllStringSubmatch(src.String(), -1) { - declared[d[1]] = true - } - - // B1: validate every total this script's committed manifest holds a - // block for, plus this job's own PR Risk matrix size and (for - // test-proxied-cmd) the Bazel lane's own shard_count — not just - // whichever total happens to equal this PR Risk job's matrix. Before - // F2 those always coincided; now the Bazel-only bazel-proxied lane - // reads a manifest block (30) that no PR-Risk-matrix-only check ever - // exercises, so a fork PR (which never runs bazel-proxied) could - // corrupt that block and still merge green. Looping over every - // distinct total in the manifest closes that gap for this script and - // any other script that later grows a second block the same way. - mm := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) - if mm == nil { - t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) - } - totalsSet := map[int]bool{} - for _, line := range strings.Split(readPolicyFile(t, root, mm[1]), "\n") { - line, _, _ = strings.Cut(line, "#") - fields := strings.Fields(line) - if len(fields) == 0 { - continue - } - if n, err := strconv.Atoi(fields[0]); err == nil { - totalsSet[n] = true - } - } - switch c.script { - case ".github/scripts/proxied-test-shard.sh": - totalsSet[bazelProxiedShardCount(t)] = true - case ".github/scripts/embedded-test-shard.sh": - // F1: the Bazel-only bazel-embedded lane reads a 50-shard cmd - // block that no PR-Risk-matrix-only (20-shard) check exercises; - // without this, "BUILD.bazel's shard_count and bazel.yml's - // check_shard_coverage.py arg both drift to a new total with no - // manifest block" passes every policy test (see review S3) -- - // the lane then silently goes 100% hash fallback and loses its - // duration balancing. - totalsSet[bazelEmbeddedCmdShardCount(t)] = true - case ".github/scripts/embedded-storage-test-shard.sh": - // F1: mirrors the cmd case above for the 15-shard storage block. - totalsSet[bazelEmbeddedStorageShardCount(t)] = true - case ".github/scripts/server-storage-test-shard.sh": - totalsSet[bazelServerFullShardCount(t)] = true - } - totals := make([]int, 0, len(totalsSet)) - for n := range totalsSet { - totals = append(totals, n) - } - sort.Ints(totals) - - for _, shards := range totals { - // The scripts' hash fallback forks per test (seconds per shard - // for the server suite): list the shards concurrently. - outs, errs := make([][]byte, shards+1), make([]error, shards+1) - var wg sync.WaitGroup - for k := 1; k <= shards; k++ { - wg.Add(1) - go func(k int) { - defer wg.Done() - cmd := exec.Command("bash", c.script, strconv.Itoa(k), strconv.Itoa(shards)) - cmd.Dir = root - cmd.Env = append(os.Environ(), "BEADS_TEST_SHARD_LIST_ONLY=1") - outs[k], errs[k] = cmd.Output() - }(k) - } - wg.Wait() - listed, manifestSum := 0, 0 - for k := 1; k <= shards; k++ { - if errs[k] != nil { - // The script itself exits 1 on a duplicate manifest entry - // or a manifest entry that names no discovered test - // (rename/typo/stale-after-delete), for the requested - // total only: this is what catches a corrupted block - // that a PR-Risk-matrix-only check at a different total - // would never see. - t.Fatalf("%s %d %d: %v\n%s", c.script, k, shards, errs[k], outs[k]) - } - for _, line := range strings.Split(string(outs[k]), "\n") { - if mc := regexp.MustCompile(`^ manifest: (\d+), fallback: \d+$`).FindStringSubmatch(line); mc != nil { - n, _ := strconv.Atoi(mc[1]) - manifestSum += n - continue - } - name, ok := strings.CutPrefix(line, " ") - if !ok || !strings.HasPrefix(name, "Test") || strings.ContainsAny(name, " :") { - continue - } - listed++ - isTest := declared[name] - switch { - case notTests[name] && isTest: - t.Errorf("%s shard %d/%d lists %s, which check_shard_coverage.py drops, but it is a real test", c.script, k, shards, name) - case !notTests[name] && !isTest: - t.Errorf("%s shard %d/%d lists %s, which is not a `func %s(t *testing.T)` test in %s: check_shard_coverage.py would report it missing on every run (add it to NOT_TESTS only if go test never runs it)", - c.script, k, shards, name, name, c.pkg) - } - } - } - if listed < 50 { - t.Errorf("%s at %d shards listed only %d tests; did the list-only output format change?", c.script, shards, listed) - } - // S1: a total that is supposed to have a committed manifest - // block (every total this loop considers does: it is either a - // live job's matrix size or a total this script's own manifest - // already names) must not have silently gone 100% hash fallback, - // which is what "the whole block was deleted" looks like from - // here: check_shard_coverage.py would still pass (it rebuilds - // its expectation from this same script), so nothing else - // catches it. - if manifestSum == 0 { - t.Errorf("%s at %d shards: manifest entries for this total sum to 0 across all shards (100%% hash fallback); its committed block in %s may have been deleted", c.script, shards, mm[1]) - } - } - } - for name := range notTests { - if name != "TestMain" { - t.Errorf("NOT_TESTS has %s; only TestMain is never a test", name) - } - } -} - -// S3: the Bazel-only 30-shard block is not frozen like the legacy 15-shard -// block (TestShardScriptsListOnlyRealTests's B1 fix catches outright -// corruption, but not a committed block that has drifted from the currently -// discovered TestProxiedServer*/TestServerMode* test set, e.g. a test added, -// renamed, or removed without anyone running --write). gen_proxied_shard_ -// manifest.py --check verifies only that the committed block names every -// discovered test exactly once -- not that its shard *assignments* match a -// fresh LPT pack -- and fails with the exact command to fix it when a name -// is missing, stale, or duplicated. It deliberately does NOT fail merely -// because proxied_test_durations.json's weights changed and the existing -// packing is now suboptimal: two PRs each adding one proxied test would -// otherwise force a full repack and conflict on unrelated shard lines (see -// --repack below for the explicit opt-in to that). Run --check here so a -// block with missing/stale/duplicate names fails go test ./scripts/... -// (//scripts:scripts_test under Bazel) instead of only -// surfacing as a test silently never running in any shard. The legacy -// 15-shard block is deliberately excluded: its header documents that it is -// frozen and must not be regenerated (see -// .github/scripts/proxied-cmd-test-shards.txt and engdocs/TESTING.md), so a -// --check against it would always fail by design. -func TestProxiedShardManifestGeneratorNotStale(t *testing.T) { - python := requireHostTool(t, "python3") - root := sourceRepoRoot(t) - cmd := exec.Command(python, "scripts/ci/gen_proxied_shard_manifest.py", "30", "--weights=duration", "--check") - cmd.Dir = root - if out, err := cmd.CombinedOutput(); err != nil { - t.Errorf("gen_proxied_shard_manifest.py 30 --weights=duration --check: %v\n%s", err, out) - } -} - // walkBazelFiles visits every BUILD.bazel and BUILD file under root (and, // with bzl, every .bzl file), skipping .git, node_modules and .beads. A // symlink to a regular file is visited: under `bazel test` on a local @@ -929,7 +735,9 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { // Committed rc files: only .bazelrc. .bazelrc.local and user.bazelrc are // developer-local (gitignored) and would be try-imported into CI runs. - for _, f := range repoFiles(t, root) { + // Under Bazel the listing is //:repo_other_files, every tracked file but + // Go and Markdown source (no rc file is either): several hundred files. + for _, f := range repoFiles(t, root, 300) { base := filepath.Base(f) if strings.Contains(base, "bazelrc") && f != ".bazelrc" && f != setupBazelActionDir+"/write-bazelrc.sh" { t.Errorf("committed rc file %s: .bazelrc's try-import would load it into every CI run", f) @@ -1098,7 +906,7 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { // selection is the point (a required-suite contract that checks it), and // nothing leaves the lanes. Pinned to exactly these args. extraRunVariants := map[string][]string{ - // The doc-freshness suite (also run by //scripts:scripts_test) under + // The doc-freshness suite (also run by //scripts:shell_scripts_test) under // -required-suite, as pr.yml's former Linux doc-freshness leg ran it. "//scripts:doc_freshness_required_test": { "-test.count=1", diff --git a/scripts/prlintmake/BUILD.bazel b/scripts/prlintmake/BUILD.bazel index e1fd0fd90..c36ad8c02 100644 --- a/scripts/prlintmake/BUILD.bazel +++ b/scripts/prlintmake/BUILD.bazel @@ -26,12 +26,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -40,7 +79,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index d907c8f9a..80b269077 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -11,8 +11,6 @@ import ( "testing" ) -const doltSQLServerImage = "dolthub/dolt-sql-server:2.2.0" - func TestPullDoltImageRetriesTransientFailures(t *testing.T) { tests := []struct { name string @@ -52,34 +50,6 @@ func TestPullDoltImageRetriesTransientFailures(t *testing.T) { } } -func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { - // No workflow pulls the image: every Dolt suite runs on hermetic dolt - // sql-servers (bazel.yml's dolt-server lanes) and needs no docker, since - // pr-risk.yml's container-backed legacy tiers were retired - // (ga-96smfk.22). The helper stays for local container-backed runs. - wantCalls := map[string]int{} - - workflowsDir := filepath.Join(sourceRepoRoot(t), ".github", "workflows") - workflowPaths, err := filepath.Glob(filepath.Join(workflowsDir, "*.y*ml")) - if err != nil { - t.Fatal(err) - } - for _, path := range workflowPaths { - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - text := string(data) - name := filepath.Base(path) - if got, want := strings.Count(text, "run: ./scripts/ci/pull-dolt-image.sh"), wantCalls[name]; got != want { - t.Errorf("%s retry-helper calls = %d, want %d", name, got, want) - } - if strings.Contains(text, "docker pull "+doltSQLServerImage) { - t.Errorf("%s still pulls the Dolt image without retries", name) - } - } -} - func TestPullDoltImageHelperIsExecutable(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Windows does not preserve Unix executable bits") diff --git a/scripts/repo_helpers_test.go b/scripts/repo_helpers_test.go index c44695897..f01b8abdf 100644 --- a/scripts/repo_helpers_test.go +++ b/scripts/repo_helpers_test.go @@ -1,11 +1,15 @@ package scripts_test -// Helpers shared by the package's test files, kept apart so a focused -// go_test over a few of them (//scripts:doc_freshness_required_test) compiles -// without the rest of the package. +// Helpers shared by the package's test files, kept apart so that each go_test +// in scripts/BUILD.bazel compiles only the files whose tests it runs (and so +// declares only the data those tests read). Helpers only: every target +// compiles this file, so a test here would run in all of them. import ( "errors" + "go/ast" + "go/token" + "os" "os/exec" "path/filepath" "runtime" @@ -22,7 +26,7 @@ func sourceRepoRoot(t *testing.T) string { t.Fatal("runtime.Caller failed") } // Under Bazel the caller path is workspace-relative; CallerDir rebuilds it - // under the runfiles root, which holds the files scripts_test declares. + // under the runfiles root, which holds the files the go_test declares. return filepath.Dir(bazeltest.CallerDir(file, "scripts")) } @@ -45,3 +49,142 @@ func exitCode(err error) int { } return -1 } + +// bazelPolicyRoot returns the repository root holding the Bazel policy files. +// Under `bazel test` those files are declared as data (//:bazel_policy_files) +// and resolved from the runfiles tree; under `go test` the source checkout is +// used directly. +func bazelPolicyRoot(t *testing.T) string { + t.Helper() + if srcdir := os.Getenv("TEST_SRCDIR"); srcdir != "" { + workspace := os.Getenv("TEST_WORKSPACE") + if workspace == "" { + workspace = "_main" + } + return filepath.Join(srcdir, workspace) + } + return sourceRepoRoot(t) +} + +func readPolicyFile(t *testing.T, root, name string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + return string(data) +} + +const ( + forkCacheEndpoint = "grpc" + "s://rbe-cache.ops.gascity.com:8443" + forkCacheInstance = "oss" + // zstd cache transfers: only the anonymous fork cache can advertise a + // compressor, so only fork-cache may ask for one, and only while + // write-bazelrc.sh's cache-zstd-probe.sh finds it advertised (a probe, + // not a repository variable: fork pull_request runs see no vars). + // Trusted remote-exec and rbe-fork never: their schedulers advertise + // none, and Bazel then refuses the remote. + forkCacheZstdLine = "build:fork-cache --remote_cache_compression" +) + +func gitRepoAvailable(root string) bool { + if _, err := exec.LookPath("git"); err != nil { + return false + } + return exec.Command("git", "-C", root, "rev-parse", "--is-inside-work-tree").Run() == nil +} + +// repoFiles lists the repository's tracked files under root, repo-relative and +// slash-separated, each a regular file (or, in a local runfiles tree, a +// symlink to one). Under `go test` in a git checkout that is `git ls-files` +// less what the working tree no longer holds; under Bazel it is the runfiles +// tree, whose repository part is what the go_test declares: //:repo_files +// (every tracked file outside .bazelignore, tools/bazel/go_srcs.py) or one of +// its partitions; elsewhere every file under root. A listing of fewer than +// minFiles files fails the test: an empty or partial tree would pass every +// scan. +func repoFiles(t *testing.T, root string, minFiles int) []string { + t.Helper() + var files []string + if bazeltest.IsBazel() || !gitRepoAvailable(root) { + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !isFileOrFileLink(path, d) { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + files = append(files, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } else { + out, err := exec.Command("git", "-C", root, "ls-files", "-z").Output() + if err != nil { + t.Fatalf("git ls-files: %v", err) + } + for _, rel := range strings.Split(strings.TrimRight(string(out), "\x00"), "\x00") { + info, err := os.Lstat(filepath.Join(root, filepath.FromSlash(rel))) + if err != nil || !info.Mode().IsRegular() { + continue // deleted in the worktree, submodule, or symlink + } + files = append(files, rel) + } + } + if len(files) < minFiles { + t.Fatalf("found only %d repository files under %s; the listing is broken", len(files), root) + } + return files +} + +// isFileOrFileLink reports whether a WalkDir entry is a regular file or a +// symlink to one. Bazel's local runfiles trees are symlink forests, so a walk +// that skipped symlinks would see no file there; a symlink to a directory +// (Bazel's bazel-* convenience links in a checkout) is never followed. +func isFileOrFileLink(path string, d os.DirEntry) bool { + if d.Type().IsRegular() { + return true + } + if d.Type()&os.ModeSymlink == 0 { + return false + } + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() +} + +// bazelRuleBlock returns the text of the top-level rule named name in a +// BUILD file, or "" if there is none. +func bazelRuleBlock(build, name string) string { + i := strings.Index(build, "\n name = \""+name+"\",\n") + if i < 0 { + return "" + } + start := strings.LastIndex(build[:i], "\n") + 1 + end := strings.Index(build[i:], "\n)\n") + if end < 0 { + return build[start:] + } + return build[start : i+end+3] +} + +func exits(body *ast.BlockStmt) bool { + for _, st := range body.List { + switch st := st.(type) { + case *ast.ReturnStmt: + return true + case *ast.BranchStmt: + if st.Tok == token.CONTINUE { + return true + } + } + } + return false +} + +const doltSQLServerImage = "dolthub/dolt-sql-server:2.2.0" diff --git a/scripts/repochecks/BUILD.bazel b/scripts/repochecks/BUILD.bazel index bc588c140..7940a4538 100644 --- a/scripts/repochecks/BUILD.bazel +++ b/scripts/repochecks/BUILD.bazel @@ -12,6 +12,12 @@ load("//tools/bazel:repo_subset.bzl", "repo_subset") # tools/bazel/check_testcases.py's Go testcase count (--not-go # //scripts/repochecks:all in bazel.yml's integration lane). # +# Each slice selects from the narrowest partition of the checkout that holds +# its files (tools/bazel/go_srcs.py: //:repo_go_srcs, //:repo_go_test_srcs, +# //:repo_doc_files, //:repo_other_files), so the guards that read no Go or +# no Markdown are not reverse dependencies of a Go-only or docs-only change. +# Only the guards that read every kind of file select from //:repo_files. +# # What each slice holds must cover what its guard reads: a renamed input fails # the build (repo_subset rejects a pattern that matches nothing), and a guard # over a slice missing a file it names fails rather than passing (e.g. @@ -21,7 +27,12 @@ load("//tools/bazel:repo_subset.bzl", "repo_subset") # checkout layout and --env (testdata/assert_tree.sh says what it expects). repo_subset( name = "run_check_fixture", - srcs = ["//:repo_files"], + srcs = [ + # This package's own files, its Markdown too: testdata/skip.md must + # be there to be left out. + ":repo_doc_files", + ":repo_other_files", + ], include = [ "scripts/repochecks/testdata/*.txt", "scripts/repochecks/testdata/assert_tree.sh", @@ -53,7 +64,7 @@ sh_test( # pathspecs, as a slice. repo_subset( name = "build_tags_files", - srcs = ["//:repo_files"], + srcs = ["//:repo_other_files"], include = [ "**/*.sh", ".bazelrc", @@ -78,7 +89,7 @@ sh_test( repo_subset( name = "workflow_files", - srcs = ["//:repo_files"], + srcs = ["//:repo_other_files"], include = [".github/workflows/*.yml"], ) @@ -134,7 +145,10 @@ sh_test( repo_subset( name = "winget_files", - srcs = ["//:repo_files"], + srcs = [ + "//:winget_manifests", + "//scripts:repo_other_files", + ], include = [ "scripts/check-winget-portable-alias.sh", "winget/*.installer.yaml", @@ -143,7 +157,7 @@ repo_subset( # check-winget-portable-alias.sh: every nested-portable winget installer # carries PortableCommandAlias: bd (GH#4908). yq comes from the executor, as -# for //scripts:scripts_test. +# for //scripts:shell_scripts_test. sh_test( name = "winget_portable_alias_test", srcs = ["run_check.sh"], @@ -158,7 +172,10 @@ sh_test( repo_subset( name = "migration_files", - srcs = ["//:repo_files"], + srcs = [ + "//internal/storage/schema:repo_other_files", + "//scripts:repo_other_files", + ], include = [ "internal/storage/schema/migrations/**", "scripts/check-migration-hygiene.sh", @@ -183,7 +200,11 @@ sh_test( repo_subset( name = "go_files", - srcs = ["//:repo_files"], + srcs = [ + "//:repo_go_srcs", + "//:repo_go_test_srcs", + "//scripts:repo_other_files", + ], include = [ "**/*.go", "scripts/ci/fmt-check.sh", @@ -284,7 +305,12 @@ sh_test( repo_subset( name = "workapi_files", - srcs = ["//:repo_files"], + srcs = [ + # The guard greps internal/workapi's .go files only. + "//:repo_go_srcs", + "//:repo_go_test_srcs", + "//scripts:repo_other_files", + ], include = [ "internal/workapi/**", "scripts/check-workapi-frontend-boundary.sh", @@ -332,12 +358,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -346,7 +411,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/repro-dolt-hang/BUILD.bazel b/scripts/repro-dolt-hang/BUILD.bazel index 9c219dbf9..7280ecc0a 100644 --- a/scripts/repro-dolt-hang/BUILD.bazel +++ b/scripts/repro-dolt-hang/BUILD.bazel @@ -20,12 +20,51 @@ go_binary( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -34,7 +73,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/repro-dolt-prod-timeouts/BUILD.bazel b/scripts/repro-dolt-prod-timeouts/BUILD.bazel index ee97e8e8f..7a807129c 100644 --- a/scripts/repro-dolt-prod-timeouts/BUILD.bazel +++ b/scripts/repro-dolt-prod-timeouts/BUILD.bazel @@ -36,12 +36,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -50,7 +89,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/scripts_targets_test.go b/scripts/scripts_targets_test.go new file mode 100644 index 000000000..a35fe17ff --- /dev/null +++ b/scripts/scripts_targets_test.go @@ -0,0 +1,105 @@ +package scripts_test + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// This package's tests are split over several go_test targets in +// scripts/BUILD.bazel, each compiling the files whose tests it runs and +// declaring only the data they read. Their srcs are kept by hand (gazelle +// would put every file back into scripts_test), so nothing else notices a new +// _test.go file in no target (its tests would never run under Bazel) or a file +// in two (its tests would run twice, one copy without the data it reads). + +var ( + scriptsGoTestRuleRe = regexp.MustCompile(`(?ms)^go_test\(\n(.*?)^\)$`) + scriptsRuleNameRe = regexp.MustCompile(`(?m)^ name = "([^"]+)",$`) + scriptsRuleSrcsRe = regexp.MustCompile(`(?ms)^ srcs = \[(.*?)\],`) + scriptsQuotedRe = regexp.MustCompile(`"([^"]+)"`) + scriptsTestFuncRe = regexp.MustCompile(`(?m)^func (Test\w*)\(\w+ \*testing\.T\) \{`) +) + +// scriptsTargetProblems checks build's go_test targets against tests, each +// _test.go file's top-level Test functions. A target whose args hold +// -required-suite is a second, selective run over tests another target runs +// in full (TestBazelRetiredLanesCannotBeNarrowed pins those) and is ignored. +func scriptsTargetProblems(build string, tests map[string][]string) []string { + var problems []string + owners := map[string][]string{} + for _, m := range scriptsGoTestRuleRe.FindAllStringSubmatch(build, -1) { + rule := m[1] + if strings.Contains(rule, "-required-suite") { + continue + } + name := scriptsRuleNameRe.FindStringSubmatch(rule) + srcs := scriptsRuleSrcsRe.FindStringSubmatch(rule) + if name == nil || srcs == nil { + problems = append(problems, "a go_test has no parsable name or srcs list:\n"+rule) + continue + } + for _, src := range scriptsQuotedRe.FindAllStringSubmatch(srcs[1], -1) { + if _, ok := tests[src[1]]; !ok { + problems = append(problems, name[1]+" lists "+src[1]+", which is not a _test.go file here") + continue + } + owners[src[1]] = append(owners[src[1]], name[1]) + } + } + files := make([]string, 0, len(tests)) + for f := range tests { + files = append(files, f) + } + sort.Strings(files) + for _, f := range files { + switch targets := owners[f]; { + case len(targets) == 0: + problems = append(problems, f+" is in no go_test: its tests never run under Bazel; add it to the target that declares what they read") + case len(targets) > 1 && len(tests[f]) > 0: + problems = append(problems, f+" defines tests ("+strings.Join(tests[f], ", ")+") and is in "+strings.Join(targets, " and ")+": they would run in each; move shared helpers to a helpers-only file") + } + } + return problems +} + +func TestScriptsTestTargetsPartitionTheTests(t *testing.T) { + fixture := "go_test(\n name = \"a_test\",\n srcs = [\n \"a_test.go\",\n \"helpers_test.go\",\n ],\n)\n\n" + + "go_test(\n name = \"b_test\",\n srcs = [\n \"a_test.go\",\n \"helpers_test.go\",\n ],\n)\n\n" + + "go_test(\n name = \"v_test\",\n srcs = [\"c_test.go\"],\n args = [\"-required-suite=x\"],\n)\n" + got := scriptsTargetProblems(fixture, map[string][]string{ + "a_test.go": {"TestA"}, + "c_test.go": {"TestC"}, + "helpers_test.go": nil, + }) + if len(got) != 2 || !strings.HasPrefix(got[0], "a_test.go defines tests (TestA) and is in a_test and b_test") || !strings.HasPrefix(got[1], "c_test.go is in no go_test") { + t.Errorf("fixture problems = %q; want a_test.go in two targets and c_test.go in none (helpers_test.go may be shared)", got) + } + + root := sourceRepoRoot(t) + paths, err := filepath.Glob(filepath.Join(root, "scripts", "*_test.go")) + if err != nil { + t.Fatal(err) + } + if len(paths) < 40 { + t.Fatalf("found only %d scripts/*_test.go files; the listing is broken", len(paths)) + } + tests := map[string][]string{} + for _, p := range paths { + data, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + var names []string + for _, m := range scriptsTestFuncRe.FindAllStringSubmatch(string(data), -1) { + names = append(names, m[1]) + } + tests[filepath.Base(p)] = names + } + for _, p := range scriptsTargetProblems(readPolicyFile(t, root, "scripts/BUILD.bazel"), tests) { + t.Errorf("scripts/BUILD.bazel: %s", p) + } +} diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go new file mode 100644 index 000000000..bbc953180 --- /dev/null +++ b/scripts/shard_manifest_test.go @@ -0,0 +1,251 @@ +package scripts_test + +// The shard manifests and shard scripts against the Go tests they shard: +// //scripts:go_test_sources_test, which reads the _test.go files of the +// sharded packages (and no non-test Go source), so an edit to a workflow file, +// a doc or a non-test Go file leaves it cached. + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "testing" +) + +// S3 (F1, mirroring F2's TestProxiedShardManifestGeneratorNotStale below — +// see that test's doc comment for the full --check rationale, not repeated +// here): the Bazel-only 50-shard cmd block and 15-shard storage block are +// not frozen like their files' legacy 20- and 5-shard blocks. +// gen_embedded_{cmd,storage}_shard_manifest.py --check verifies only that +// the committed block names every discovered test exactly once, failing with +// the exact command to fix it when a name is missing, stale, or duplicated — +// run here so a drifted block fails `go test ./scripts/...` +// (//scripts:go_test_sources_test under Bazel) instead of only surfacing as +// a test silently never running in any shard. The legacy blocks are +// deliberately excluded: both files document that their 20- and 5-shard +// blocks are frozen (see .github/scripts/embedded-{cmd,storage}- +// test-shards.txt and engdocs/TESTING.md), so a --check against them is +// expected to report "missing" entries by design (see those generators' +// module docstrings) and is not what this test runs. +func TestCmdEmbeddedShardManifestGeneratorNotStale(t *testing.T) { + python := requireHostTool(t, "python3") + root := sourceRepoRoot(t) + shards := strconv.Itoa(bazelEmbeddedCmdShardCount(t)) + cmd := exec.Command(python, "scripts/ci/gen_embedded_cmd_shard_manifest.py", shards, "--weights=duration", "--check") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Errorf("gen_embedded_cmd_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) + } +} + +// TestStorageEmbeddedShardManifestGeneratorNotStale mirrors +// TestCmdEmbeddedShardManifestGeneratorNotStale above for the storage tier's +// Bazel-only 15-shard block; see that test's doc comment. +func TestStorageEmbeddedShardManifestGeneratorNotStale(t *testing.T) { + python := requireHostTool(t, "python3") + root := sourceRepoRoot(t) + shards := strconv.Itoa(bazelEmbeddedStorageShardCount(t)) + cmd := exec.Command(python, "scripts/ci/gen_embedded_storage_shard_manifest.py", shards, "--weights=duration", "--check") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Errorf("gen_embedded_storage_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) + } +} + +// Review G1: the checker's input is the real shard scripts' list-only +// output (every retired lane's: the embedded, proxied and server suites). Every name they list, for every shard, must be a test go test +// runs (declared `func Name(t *testing.T)` in the package's _test.go files), +// or one check_shard_coverage.py drops as NOT_TESTS; and each NOT_TESTS name +// must really not be a test (TestMain takes *testing.M). Otherwise the +// checker reports a listed test that "did not run" on every real run. +func TestShardScriptsListOnlyRealTests(t *testing.T) { + // The CI shard scripts use bash 4 associative arrays (Linux runners only). + requireAutofixBash(t) + root := sourceRepoRoot(t) + m := regexp.MustCompile(`(?m)^NOT_TESTS = frozenset\(\{([^}]*)\}\)`).FindStringSubmatch(readPolicyFile(t, root, "tools/bazel/check_shard_coverage.py")) + if m == nil { + t.Fatal("tools/bazel/check_shard_coverage.py has no NOT_TESTS = frozenset({...})") + } + notTests := map[string]bool{} + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(m[1], -1) { + notTests[q[1]] = true + } + for _, c := range []struct{ job, script, pkg string }{ + {"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd"}, + {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt"}, + // D2 step 2: the proxied and server lanes' suites. + {"test-proxied-cmd", ".github/scripts/proxied-test-shard.sh", "cmd/bd"}, + {"test-server-storage-full", ".github/scripts/server-storage-test-shard.sh", "internal/storage/dolt"}, + } { + var src strings.Builder + files, err := filepath.Glob(filepath.Join(root, c.pkg, "*_test.go")) + if err != nil || len(files) == 0 { + t.Fatalf("%s: no _test.go files (%v)", c.pkg, err) + } + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + src.Write(data) + src.WriteString("\n") + } + declared := map[string]bool{} + for _, d := range regexp.MustCompile(`(?m)^func (Test\w*)\(\w+ \*testing\.T\) \{`).FindAllStringSubmatch(src.String(), -1) { + declared[d[1]] = true + } + + // B1: validate every total this script's committed manifest holds a + // block for, plus this job's own PR Risk matrix size and (for + // test-proxied-cmd) the Bazel lane's own shard_count — not just + // whichever total happens to equal this PR Risk job's matrix. Before + // F2 those always coincided; now the Bazel-only bazel-proxied lane + // reads a manifest block (30) that no PR-Risk-matrix-only check ever + // exercises, so a fork PR (which never runs bazel-proxied) could + // corrupt that block and still merge green. Looping over every + // distinct total in the manifest closes that gap for this script and + // any other script that later grows a second block the same way. + mm := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) + if mm == nil { + t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) + } + totalsSet := map[int]bool{} + for _, line := range strings.Split(readPolicyFile(t, root, mm[1]), "\n") { + line, _, _ = strings.Cut(line, "#") + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + if n, err := strconv.Atoi(fields[0]); err == nil { + totalsSet[n] = true + } + } + switch c.script { + case ".github/scripts/proxied-test-shard.sh": + totalsSet[bazelProxiedShardCount(t)] = true + case ".github/scripts/embedded-test-shard.sh": + // F1: the Bazel-only bazel-embedded lane reads a 50-shard cmd + // block that no PR-Risk-matrix-only (20-shard) check exercises; + // without this, "BUILD.bazel's shard_count and bazel.yml's + // check_shard_coverage.py arg both drift to a new total with no + // manifest block" passes every policy test (see review S3) -- + // the lane then silently goes 100% hash fallback and loses its + // duration balancing. + totalsSet[bazelEmbeddedCmdShardCount(t)] = true + case ".github/scripts/embedded-storage-test-shard.sh": + // F1: mirrors the cmd case above for the 15-shard storage block. + totalsSet[bazelEmbeddedStorageShardCount(t)] = true + case ".github/scripts/server-storage-test-shard.sh": + totalsSet[bazelServerFullShardCount(t)] = true + } + totals := make([]int, 0, len(totalsSet)) + for n := range totalsSet { + totals = append(totals, n) + } + sort.Ints(totals) + + for _, shards := range totals { + // The scripts' hash fallback forks per test (seconds per shard + // for the server suite): list the shards concurrently. + outs, errs := make([][]byte, shards+1), make([]error, shards+1) + var wg sync.WaitGroup + for k := 1; k <= shards; k++ { + wg.Add(1) + go func(k int) { + defer wg.Done() + cmd := exec.Command("bash", c.script, strconv.Itoa(k), strconv.Itoa(shards)) + cmd.Dir = root + cmd.Env = append(os.Environ(), "BEADS_TEST_SHARD_LIST_ONLY=1") + outs[k], errs[k] = cmd.Output() + }(k) + } + wg.Wait() + listed, manifestSum := 0, 0 + for k := 1; k <= shards; k++ { + if errs[k] != nil { + // The script itself exits 1 on a duplicate manifest entry + // or a manifest entry that names no discovered test + // (rename/typo/stale-after-delete), for the requested + // total only: this is what catches a corrupted block + // that a PR-Risk-matrix-only check at a different total + // would never see. + t.Fatalf("%s %d %d: %v\n%s", c.script, k, shards, errs[k], outs[k]) + } + for _, line := range strings.Split(string(outs[k]), "\n") { + if mc := regexp.MustCompile(`^ manifest: (\d+), fallback: \d+$`).FindStringSubmatch(line); mc != nil { + n, _ := strconv.Atoi(mc[1]) + manifestSum += n + continue + } + name, ok := strings.CutPrefix(line, " ") + if !ok || !strings.HasPrefix(name, "Test") || strings.ContainsAny(name, " :") { + continue + } + listed++ + isTest := declared[name] + switch { + case notTests[name] && isTest: + t.Errorf("%s shard %d/%d lists %s, which check_shard_coverage.py drops, but it is a real test", c.script, k, shards, name) + case !notTests[name] && !isTest: + t.Errorf("%s shard %d/%d lists %s, which is not a `func %s(t *testing.T)` test in %s: check_shard_coverage.py would report it missing on every run (add it to NOT_TESTS only if go test never runs it)", + c.script, k, shards, name, name, c.pkg) + } + } + } + if listed < 50 { + t.Errorf("%s at %d shards listed only %d tests; did the list-only output format change?", c.script, shards, listed) + } + // S1: a total that is supposed to have a committed manifest + // block (every total this loop considers does: it is either a + // live job's matrix size or a total this script's own manifest + // already names) must not have silently gone 100% hash fallback, + // which is what "the whole block was deleted" looks like from + // here: check_shard_coverage.py would still pass (it rebuilds + // its expectation from this same script), so nothing else + // catches it. + if manifestSum == 0 { + t.Errorf("%s at %d shards: manifest entries for this total sum to 0 across all shards (100%% hash fallback); its committed block in %s may have been deleted", c.script, shards, mm[1]) + } + } + } + for name := range notTests { + if name != "TestMain" { + t.Errorf("NOT_TESTS has %s; only TestMain is never a test", name) + } + } +} + +// S3: the Bazel-only 30-shard block is not frozen like the legacy 15-shard +// block (TestShardScriptsListOnlyRealTests's B1 fix catches outright +// corruption, but not a committed block that has drifted from the currently +// discovered TestProxiedServer*/TestServerMode* test set, e.g. a test added, +// renamed, or removed without anyone running --write). gen_proxied_shard_ +// manifest.py --check verifies only that the committed block names every +// discovered test exactly once -- not that its shard *assignments* match a +// fresh LPT pack -- and fails with the exact command to fix it when a name +// is missing, stale, or duplicated. It deliberately does NOT fail merely +// because proxied_test_durations.json's weights changed and the existing +// packing is now suboptimal: two PRs each adding one proxied test would +// otherwise force a full repack and conflict on unrelated shard lines (see +// --repack below for the explicit opt-in to that). Run --check here so a +// block with missing/stale/duplicate names fails go test ./scripts/... +// (//scripts:go_test_sources_test under Bazel) instead of only +// surfacing as a test silently never running in any shard. The legacy +// 15-shard block is deliberately excluded: its header documents that it is +// frozen and must not be regenerated (see +// .github/scripts/proxied-cmd-test-shards.txt and engdocs/TESTING.md), so a +// --check against it would always fail by design. +func TestProxiedShardManifestGeneratorNotStale(t *testing.T) { + python := requireHostTool(t, "python3") + root := sourceRepoRoot(t) + cmd := exec.Command(python, "scripts/ci/gen_proxied_shard_manifest.py", "30", "--weights=duration", "--check") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Errorf("gen_proxied_shard_manifest.py 30 --weights=duration --check: %v\n%s", err, out) + } +} diff --git a/scripts/testing_short_tree_test.go b/scripts/testing_short_tree_test.go new file mode 100644 index 000000000..ffac3fb23 --- /dev/null +++ b/scripts/testing_short_tree_test.go @@ -0,0 +1,33 @@ +package scripts_test + +// check-testing-short.sh over the whole Go tree: //scripts:go_sources_test, +// which declares every .go file (test and non-test) and nothing else. The +// script's own fixture tests are in check_testing_short_test.go. + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" +) + +func TestCheckTestingShortPassesOnCleanRepoTree(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("checker is a Bash boundary") + } + repo := sourceRepoRoot(t) + // Under Bazel the tree is //:repo_go_srcs and //:repo_go_test_srcs; a tree + // missing the allowlisted call sites would let the scan pass vacuously. + for _, rel := range []string{"internal/hooks/hooks_test.go", "internal/storage/dolt/concurrent_test.go", "internal/workapi/sweep_test.go"} { + if _, err := os.Stat(filepath.Join(repo, filepath.FromSlash(rel))); err != nil { + t.Fatalf("the scanned tree lacks %s, which holds an allowlisted call: %v", rel, err) + } + } + cmd := exec.Command("bash", filepath.Join(repo, "scripts", "check-testing-short.sh")) + cmd.Dir = repo + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("check-testing-short.sh failed on clean repo tree (all 9 allowlist entries must still be recognized): %v\noutput: %s", err, out) + } +} diff --git a/scripts/tracked_files_test.go b/scripts/tracked_files_test.go new file mode 100644 index 000000000..1fd09c3af --- /dev/null +++ b/scripts/tracked_files_test.go @@ -0,0 +1,295 @@ +package scripts_test + +import ( + "bufio" + "bytes" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// Scans of every tracked file: //scripts:tracked_files_test, the one target +// here that declares //:repo_files (the whole checkout), so that every other +// target can declare only the partition it reads and stay cached when an +// unrelated file changes. Keep this file to tests that really read every +// tracked file, and keep them cheap: any edit anywhere re-runs them. + +// trackedFilesMin is the fewest files a listing of //:repo_files may hold +// (the checkout has several thousand). +const trackedFilesMin = 1000 + +// --- no remote-execution endpoints in tracked files ------------------------- + +// Remote-execution endpoints, credentials and TLS material belong in a +// gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the +// workspace, never in the repository. The scan covers committed bazelrc-like +// files, Markdown docs and .github/** (plan §3.8) and flags: +// - any remote/BES/TLS flag below whose value is a literal (not a +// , a $VARIABLE/${{ expression }}, empty, or loopback); +// - in rc and .github files, any gRPC URL that is not loopback. Markdown is +// exempt from this rule because beads documents OTel gRPC exporters. +// +// The one exception is publicForkCacheLines. + +var ( + remoteFlagNames = `remote_executor|remote_cache|remote_downloader|remote_header|remote_instance_name|` + + `bes_backend|bes_results_url|bes_header|tls_client_certificate|tls_client_key|tls_certificate` + // `--flag=value` is recognized everywhere; `--flag value` only in rc and + // workflow files, where prose ("--remote_executor and ...") does not occur. + remoteFlagEqRe = regexp.MustCompile(`--(` + remoteFlagNames + `)=("[^"]*"|'[^']*'|[^\s"'` + "`" + `]*)`) + remoteFlagSpaceRe = regexp.MustCompile(`--(` + remoteFlagNames + `)\s+("[^"]*"|'[^']*'|[^\s"'` + "`" + `]+)`) + remoteEndpointRe = regexp.MustCompile(`\bgrpcs?://[^\s"'<>)\]]+`) + loopbackValueRe = regexp.MustCompile(`^(?:[a-z][a-z0-9+.-]*://)?(?:127\.0\.0\.1|localhost|\[::1\])(?:[:/]|$)`) + // remoteScanPrefilter: a file lacking all of these cannot produce a hit. + remoteScanPrefilter = [][]byte{[]byte("--remote_"), []byte("--bes_"), []byte("--tls_"), []byte("grpc")} +) + +// publicForkCacheLines: the endpoint lines of .bazelrc's fork-cache config, +// public by design. rbe-west's rbe-cache endpoint is anonymous and read-only +// (action-cache and CAS reads; writes and Execute are refused by the farm), +// so there is no credential to leak. Exactly these lines, trimmed, and only +// in .bazelrc; the same line anywhere else, or any other endpoint, is still a +// hit. Assembled so this file never contains a literal endpoint. +var publicForkCacheLines = map[string]bool{ + "build:fork-cache --remote_cache=" + forkCacheEndpoint: true, + "build:fork-cache --remote_instance_name=" + forkCacheInstance: true, +} + +// publicRBEForkPin: setup-bazel's fork-credential.sh pins the one endpoint +// rbe-fork-mint may hand a fork run (rbe-fork, :8444), so a compromised +// mint cannot point fork builds elsewhere. It is public and carries no +// credential: allowed as exactly this line, in exactly that file (a byte +// copy of gascity's tools/rbe/fork-credential.sh). +const ( + publicRBEForkPinFile = ".github/actions/setup-bazel/fork-credential.sh" + publicRBEForkPinLine = `ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-fork\.ops\.gascity\.com:8444$'}` +) + +type endpointHit struct { + path string + line int + what string +} + +// remoteScanKind classifies a tracked path for the endpoint scan. +func remoteScanKind(rel string) (scan, strict bool) { + base := filepath.Base(rel) + switch { + case strings.Contains(base, "bazelrc"), strings.HasPrefix(rel, ".github/"): + return true, true + case strings.HasSuffix(base, ".md"): + return true, false + } + return false, false +} + +func allowedRemoteValue(v string) bool { + v = strings.Trim(v, `"'`) + if i := strings.Index(v, "://"); i >= 0 { + v = v[i+len("://"):] + } + return v == "" || strings.HasPrefix(v, "<") || strings.HasPrefix(v, "$") || loopbackValueRe.MatchString(v) +} + +// findRemoteEndpoints reports literal remote endpoints/credentials in content. +// strict enables the whitespace-separated flag form and the gRPC URL rule. +func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit { + if bytes.IndexByte(content, 0) >= 0 { // binary + return nil + } + relevant := false + for _, needle := range remoteScanPrefilter { + if bytes.Contains(content, needle) { + relevant = true + break + } + } + if !relevant { + return nil + } + flagRes := []*regexp.Regexp{remoteFlagEqRe} + if strict { + flagRes = append(flagRes, remoteFlagSpaceRe) + } + var hits []endpointHit + scanner := bufio.NewScanner(bytes.NewReader(content)) + scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + for n := 1; scanner.Scan(); n++ { + line := scanner.Text() + if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] { + continue + } + if path == publicRBEForkPinFile && line == publicRBEForkPinLine { + continue + } + for _, re := range flagRes { + for _, m := range re.FindAllStringSubmatch(line, -1) { + if !allowedRemoteValue(m[2]) { + hits = append(hits, endpointHit{path: path, line: n, what: m[0]}) + } + } + } + if strict { + for _, url := range remoteEndpointRe.FindAllString(line, -1) { + if !allowedRemoteValue(url) { + hits = append(hits, endpointHit{path: path, line: n, what: url}) + } + } + } + } + return hits +} + +func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) { + // Fixtures are assembled at runtime so this file never contains a literal + // endpoint itself. + scheme := "grpc" + "s://" + flag := func(name string) string { return "--" + name } + for _, bad := range []string{ + "build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe.example.com:443\n", + "build:remote-exec " + flag("remote_executor") + "=rbe.example.com:443\n", + "build:remote-exec " + flag("remote_executor") + " rbe.example.com:443\n", + "build " + flag("remote_cache") + "=https://cache.example.com\n", + "build " + flag("bes_backend") + "=bes.example.com\n", + "build " + flag("bes_results_url") + "=https://results.example.com/inv/\n", + "build " + flag("remote_header") + "=x-api-key=abc123\n", + "build " + flag("tls_client_certificate") + "=/etc/rbe/client.crt\n", + "build " + flag("tls_client_key") + "=\"/etc/rbe/client.key\"\n", + "# farm: " + scheme + "10.0.0.5:8980\n", + "REMOTE=" + "grpc" + "://cache.internal:9092\n", + } { + if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 { + t.Errorf("endpoint fixture not detected: %q", bad) + } + } + for _, ok := range []string{ + "build:remote-exec " + flag("remote_executor") + "=" + scheme + "\n", + "build:remote-exec " + flag("remote_executor") + "=\n", + "build:remote-exec " + flag("tls_client_certificate") + "=\n", + " run: echo \"build " + flag("remote_executor") + "=${{ secrets.BAZEL_REMOTE_EXECUTOR }}\" >> \"$RUNNER_TEMP/ci.bazelrc\"\n", + " " + flag("tls_client_key") + " \"$RUNNER_TEMP/rbe.key\"\n", + " " + flag("remote_executor") + "=" + scheme + "${{ secrets.RBE_HOST }}\n", + flag("remote_cache") + "=" + "grpc" + "://127.0.0.1:50052\n", + flag("remote_cache") + "=localhost:9092\n", + flag("remote_cache") + "=\n", + "build " + flag("remote_timeout") + "=600\n", + "use grpc for transport\n", + } { + if hits := findRemoteEndpoints(".bazelrc", []byte(ok), true); len(hits) != 0 { + t.Errorf("allowed fixture flagged: %q -> %v", ok, hits) + } + } + // The fork cache's public endpoint: exactly its two .bazelrc lines. + forkCache := "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n" + + " build:fork-cache " + flag("remote_instance_name") + "=oss \n" + if hits := findRemoteEndpoints(".bazelrc", []byte(forkCache), true); len(hits) != 0 { + t.Errorf("fork-cache endpoint lines flagged in .bazelrc: %v", hits) + } + for _, path := range []string{"tools/ci.bazelrc", ".bazelrc.local", "user.bazelrc", ".github/workflows/x.yml", ".github/actions/setup-bazel/x.sh"} { + if hits := findRemoteEndpoints(path, []byte(forkCache), true); len(hits) < 2 { + t.Errorf("fork-cache endpoint lines in %s: hits %v, want both lines flagged (allowlisted in .bazelrc only)", path, hits) + } + } + for _, bad := range []string{ + "build:fork-cache " + flag("remote_cache") + "=" + scheme + "other.example:8443\n", + "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "/x\n", + "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + " " + flag("remote_header") + "=x-api-key=abc\n", + "build " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n", + "build:remote-exec " + flag("remote_executor") + "=" + forkCacheEndpoint + "\n", + "build:fork-cache " + flag("remote_instance_name") + "=beads\n", + "# see " + forkCacheEndpoint + "\n", + } { + if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 { + t.Errorf("non-allowlisted fork-cache variant not detected in .bazelrc: %q", bad) + } + } + // rbe-fork's endpoint pin: that line in that file only; anything else + // naming the endpoint, there or elsewhere, is flagged. + if hits := findRemoteEndpoints(publicRBEForkPinFile, []byte("set -eu\n"+publicRBEForkPinLine+"\n"), true); len(hits) != 0 { + t.Errorf("rbe-fork endpoint pin flagged in %s: %v", publicRBEForkPinFile, hits) + } + for _, path := range []string{".github/actions/setup-bazel/write-bazelrc.sh", ".github/workflows/bazel.yml", ".bazelrc", "tools/rbe/fork-credential.sh"} { + if len(findRemoteEndpoints(path, []byte(publicRBEForkPinLine+"\n"), true)) == 0 { + t.Errorf("rbe-fork endpoint pin not flagged in %s (allowlisted in %s only)", path, publicRBEForkPinFile) + } + } + for _, bad := range []string{ + " " + publicRBEForkPinLine, + strings.Replace(publicRBEForkPinLine, "8444", "443", 1), + `ENDPOINT=` + "grpc" + `s://rbe-fork.ops.gascity.com:8444`, + "build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe-fork.ops.gascity.com:8444", + } { + if len(findRemoteEndpoints(publicRBEForkPinFile, []byte(bad+"\n"), true)) == 0 { + t.Errorf("non-allowlisted rbe-fork line not detected in %s: %q", publicRBEForkPinFile, bad) + } + } + // Markdown: OTel gRPC exporter URLs and prose mentioning the flag are fine; + // a literal flag value is not. + for _, ok := range []string{ + "Set OTEL_EXPORTER_OTLP_ENDPOINT=" + "grpc" + "://collector:4317\n", + "Pass " + flag("remote_executor") + " and TLS flags from user.bazelrc.\n", + } { + if hits := findRemoteEndpoints("docs/x.md", []byte(ok), false); len(hits) != 0 { + t.Errorf("allowed Markdown fixture flagged: %q -> %v", ok, hits) + } + } + if len(findRemoteEndpoints("docs/x.md", []byte("bazel build "+flag("remote_executor")+"=rbe.example.com:443 //...\n"), false)) == 0 { + t.Error("literal remote_executor in Markdown not detected") + } + for rel, want := range map[string][2]bool{ + ".bazelrc": {true, true}, + "tools/ci.bazelrc": {true, true}, + ".bazelrc.local": {true, true}, + ".github/workflows/x.yml": {true, true}, + "docs/BAZEL.md": {true, false}, + "issues.jsonl": {false, false}, + "internal/telemetry/x.go": {false, false}, + } { + if scan, strict := remoteScanKind(rel); scan != want[0] || strict != want[1] { + t.Errorf("remoteScanKind(%q) = %v,%v, want %v,%v", rel, scan, strict, want[0], want[1]) + } + } + + root := bazelPolicyRoot(t) + var hits []endpointHit + for _, rel := range repoFiles(t, root, trackedFilesMin) { + scan, strict := remoteScanKind(rel) + if !scan { + continue + } + content, err := os.ReadFile(filepath.Join(root, rel)) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + hits = append(hits, findRemoteEndpoints(rel, content, strict)...) + } + for _, h := range hits { + t.Errorf("%s:%d: remote endpoint or credential %q in a tracked file; it belongs in a gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the workspace", h.path, h.line, h.what) + } +} + +// TestNoLocalPlanPathsInTrackedFiles: tracked files must not point readers +// at a maintainer's private, out-of-repo planning notes (a home-directory +// planning-notes tree), which no other contributor can open. Cite an +// in-repo doc, a bead, or a PR instead. The needle is assembled at runtime so +// this file does not match itself. +func TestNoLocalPlanPathsInTrackedFiles(t *testing.T) { + root := bazelPolicyRoot(t) + needle := []byte("beads-" + "bazel-plan") + for _, rel := range repoFiles(t, root, trackedFilesMin) { + content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + if bytes.IndexByte(content, 0) >= 0 { + continue // binary, as git grep -I + } + for n, line := range bytes.Split(content, []byte("\n")) { + if bytes.Contains(line, needle) { + t.Errorf("%s:%d: tracked file references a local, non-repo %s path; point at an in-repo doc, bead or PR instead", rel, n+1, needle) + } + } + } +} diff --git a/scripts/workflowtags/BUILD.bazel b/scripts/workflowtags/BUILD.bazel index f5ff5af79..2a50a5d03 100644 --- a/scripts/workflowtags/BUILD.bazel +++ b/scripts/workflowtags/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/test/conformance/BUILD.bazel b/test/conformance/BUILD.bazel index 4758a9c00..6ddf3ddda 100644 --- a/test/conformance/BUILD.bazel +++ b/test/conformance/BUILD.bazel @@ -32,12 +32,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -46,7 +85,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/test/docsync/BUILD.bazel b/test/docsync/BUILD.bazel index dcf713b95..ea3a13d85 100644 --- a/test/docsync/BUILD.bazel +++ b/test/docsync/BUILD.bazel @@ -8,21 +8,81 @@ go_test( "doltpin_test.go", "init_safety_recovery_test.go", ], - # The checkout as Bazel sees it: the docs trees, every link target, and - # every AGENTS.md and test file the agent-instruction checks resolve. - data = ["//:repo_files"], + # Every Markdown file (the docs trees, the root docs, every AGENTS.md), + # every Go test file (the guard-test names the agent instructions cite), + # docs.json, and the other files the docs link to by path. Not the + # checkout: a Go-only or workflow-only change leaves this cached. A link to + # a file outside these fails under Bazel; declare the file here. + data = [ + "//:.github/workflows/bazel.yml", + "//:.golangci.yml", + "//:beads.go", + "//:docs/docs.json", + "//:repo_doc_files", + "//:repo_go_test_srcs", + "//cmd/bd:capability_registry.go", + "//cmd/bd:doctor.go", + "//cmd/bd:init_safety_help.go", + "//cmd/bd:label.go", + "//cmd/bd:main.go", + "//cmd/bd:state.go", + "//internal/testutil:testdoltcommon.go", + "//scripts:bump-version.sh", + "//scripts:check-cli-docs-drift.sh", + "//scripts:generate-cli-docs.sh", + "//scripts:test.sh", + ], deps = ["//internal/testutil/bazeltest"], ) # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +91,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/test/docsync/agentfiles_test.go b/test/docsync/agentfiles_test.go index 6547dc137..b83286b25 100644 --- a/test/docsync/agentfiles_test.go +++ b/test/docsync/agentfiles_test.go @@ -82,9 +82,10 @@ func contributorAgentFiles(t *testing.T, root string) []string { // CLAUDE.md`: Claude Code skips AGENTS.md files when a CLAUDE.md exists above // them), every nested one is listed in the root routing table, and every // local link, backticked repo path, and backticked guard-test name in them -// resolves. Make targets are not checked. Under Bazel the checkout is -// //:repo_files, where a source symlink arrives as its target's content, so -// CLAUDE.md is checked by content: a copy that drifts from AGENTS.md fails. +// resolves. Make targets are not checked. Under Bazel the tree is the test's +// data (every Markdown file among it), where a source symlink arrives as its +// target's content, so CLAUDE.md is checked by content: a copy that drifts +// from AGENTS.md fails. func TestAgentInstructionFilesAreLinkedAndRouted(t *testing.T) { root := repoRoot() agentFiles := contributorAgentFiles(t, root) @@ -167,7 +168,7 @@ func TestAgentInstructionFilesAreLinkedAndRouted(t *testing.T) { } if len(broken) > 0 { sort.Strings(broken) - t.Errorf("agent instruction files reference paths that do not exist (%d):", len(broken)) + t.Errorf("agent instruction files reference paths that do not exist (%d)%s:", len(broken), declaredDataHint()) for _, b := range broken { t.Errorf(" %s", b) } diff --git a/test/docsync/docsync_test.go b/test/docsync/docsync_test.go index df734e1f9..ad81edf1b 100644 --- a/test/docsync/docsync_test.go +++ b/test/docsync/docsync_test.go @@ -31,6 +31,15 @@ func repoRoot() string { return filepath.Join(filepath.Dir(filename), "..", "..") } +// declaredDataHint is appended to a broken-path report under Bazel, where the +// test sees only the files test/docsync/BUILD.bazel declares. +func declaredDataHint() string { + if !bazeltest.IsBazel() { + return "" + } + return " (under Bazel only declared files exist: if a path below is in the checkout, add it to docsync_test's data in test/docsync/BUILD.bazel)" +} + var markdownLinkRE = regexp.MustCompile(`\[[^][]*\]\(([^)]+)\)`) // docsPublishExemptions lists markdown files under docs/ that are allowed to @@ -334,7 +343,7 @@ func TestDocsSiteLinks(t *testing.T) { if len(broken) > 0 { sort.Strings(broken) - t.Errorf("broken or misconvention links in docs/ (%d):", len(broken)) + t.Errorf("broken or misconvention links in docs/ (%d)%s:", len(broken), declaredDataHint()) for _, b := range broken { t.Errorf(" %s", b) } @@ -386,7 +395,7 @@ func TestEngdocsAndRootMarkdownLinks(t *testing.T) { if len(broken) > 0 { sort.Strings(broken) - t.Errorf("broken local markdown links (%d) — engdocs/ and root files are GitHub-viewed and need exact paths:", len(broken)) + t.Errorf("broken local markdown links (%d) — engdocs/ and root files are GitHub-viewed and need exact paths%s:", len(broken), declaredDataHint()) for _, b := range broken { t.Errorf(" %s", b) } diff --git a/test/testmainconvention/BUILD.bazel b/test/testmainconvention/BUILD.bazel index cdb841788..b7d3537b9 100644 --- a/test/testmainconvention/BUILD.bazel +++ b/test/testmainconvention/BUILD.bazel @@ -8,12 +8,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -22,7 +61,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tests/hook_timeout_backends/BUILD.bazel b/tests/hook_timeout_backends/BUILD.bazel index aef412762..ec9fa9945 100644 --- a/tests/hook_timeout_backends/BUILD.bazel +++ b/tests/hook_timeout_backends/BUILD.bazel @@ -40,12 +40,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -54,7 +93,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tests/migration/BUILD.bazel b/tests/migration/BUILD.bazel index 4d5bf1f21..36a2ffb5d 100644 --- a/tests/migration/BUILD.bazel +++ b/tests/migration/BUILD.bazel @@ -111,12 +111,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -125,7 +164,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tests/regression/BUILD.bazel b/tests/regression/BUILD.bazel index ee7410d1a..b0c7f12ca 100644 --- a/tests/regression/BUILD.bazel +++ b/tests/regression/BUILD.bazel @@ -51,12 +51,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -65,7 +104,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tests/upgrade_smoke/BUILD.bazel b/tests/upgrade_smoke/BUILD.bazel index be70457de..9963bb535 100644 --- a/tests/upgrade_smoke/BUILD.bazel +++ b/tests/upgrade_smoke/BUILD.bazel @@ -23,12 +23,51 @@ upgrade_smoke_tests( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/third_party/cc_toolchain/BUILD.bazel b/third_party/cc_toolchain/BUILD.bazel index c6e24cb8a..30c5af16d 100644 --- a/third_party/cc_toolchain/BUILD.bazel +++ b/third_party/cc_toolchain/BUILD.bazel @@ -5,12 +5,51 @@ # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -19,7 +58,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/third_party/patches/BUILD.bazel b/third_party/patches/BUILD.bazel index 086c77351..f27f8bb37 100644 --- a/third_party/patches/BUILD.bazel +++ b/third_party/patches/BUILD.bazel @@ -25,12 +25,51 @@ exports_files(glob(["*.patch"])) # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -39,7 +78,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index e35ecd946..6b17dde5c 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -13,6 +13,16 @@ _DOLT_PLATFORMS_ONLY = select({ "//conditions:default": ["@platforms//:incompatible"], }) +# Read by //scripts:go_test_sources_test: the shard-coverage check's list of +# names that are not tests, and the Bazel-skip allowlist. +exports_files( + [ + "check_shard_coverage.py", + "equivalence_allowlist.txt", + ], + visibility = ["//scripts:__pkg__"], +) + # sh_test src for a second test target over an existing go_test binary (see # the script header), e.g. //cmd/bd/doctor/fix:fix_dolt_test. exports_files( @@ -305,12 +315,51 @@ release_cross_build( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -319,7 +368,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/bazel/go_srcs.py b/tools/bazel/go_srcs.py index e4ca6b6d3..4b685968a 100755 --- a/tools/bazel/go_srcs.py +++ b/tools/bazel/go_srcs.py @@ -21,16 +21,28 @@ Blocks in packages that are no longer listed are removed. Run from the repository root after gazelle (see `make bazel-sync`). Output is deterministic and gazelle-stable, so a clean sync leaves git clean. -Every package also gets a managed `repo_files` block: a filegroup of all of -the package's own files (`glob(["**"])` less local build and editor debris), -and the root package's aggregates every package's into `//:repo_files`. That -is the checkout as Bazel sees it, the `data` of the repository-policy tests -that walk the whole tree (every BUILD file, every test file, every tracked -Markdown file). A package without the block would drop out of their view and -pass them vacuously, which is why `make bazel-sync-check` (bazel.yml's BUILD -sync step, on every PR) fails on a missing or stale block. Trees in -.bazelignore (.beads, website, the nested example modules, agent worktrees, -node_modules) are outside Bazel and so outside //:repo_files. +Every package also gets a managed `repo_files` block: four filegroups that +partition the package's own files (`glob(["**"])` less local build and editor +debris) by what kind of input they are, + + repo_go_srcs non-test Go source (**/*.go less **/*_test.go) + repo_go_test_srcs Go test source (**/*_test.go) + repo_doc_files Markdown (**/*.md, **/*.mdx) + repo_other_files everything else: BUILD and .bzl files, scripts, + workflows, configuration, testdata + +and the root package's block aggregates every package's into the +`//:repo_` filegroups, whose union is `//:repo_files`. That is the +checkout as Bazel sees it. A repository-policy test declares the narrowest of +these (or one package's partition) that covers what it reads, so that an edit +outside it leaves the test cached: a docs-only change re-runs no Go-source +scan, a Go-only change no workflow policy test. Only the tests that really +read every tracked file declare `//:repo_files`. A package without the block +would drop out of their view and pass them vacuously, which is why `make +bazel-sync-check` (bazel.yml's BUILD sync step, on every PR) fails on a +missing or stale block. Trees in .bazelignore (.beads, website, the nested +example modules, agent worktrees, node_modules) are outside Bazel and so +outside //:repo_files. tools/bazel/BUILD.bazel also gets a managed `release_cross` block: the release_cross_build (tools/bazel/release_cross.bzl) that @@ -155,13 +167,27 @@ ROOT_REPO_FILES_EXCLUDE = ( "user.bazelrc", ) -# Who may read //:repo_files: the repository-policy tests. +# Who may read //:repo_files, every tracked file: the tests that scan the +# whole checkout. Everything else declares a partition below. REPO_FILES_VISIBILITY = ( "//scripts:__pkg__", "//scripts/repochecks:__pkg__", - "//test/docsync:__pkg__", ) +# The partitions of every package's files, as (name, include, exclude): each +# file is in exactly one. The debris patterns above match no Go or Markdown +# file, so only node_modules (and the root's own excludes) are dropped from +# the Go and doc partitions. +REPO_PARTITIONS = ( + ("repo_doc_files", ("**/*.md", "**/*.mdx"), ()), + ("repo_go_srcs", ("**/*.go",), ("**/*_test.go",)), + ("repo_go_test_srcs", ("**/*_test.go",), ()), + ("repo_other_files", ("**",), ("**/*.go", "**/*.md", "**/*.mdx")), +) +OTHER_PARTITION = "repo_other_files" +# Partitions are readable by any test in the repository; //:repo_files is not. +REPO_PARTITION_VISIBILITY = "//:__subpackages__" + def packages_under(root: str) -> list[str]: found = [] @@ -203,31 +229,56 @@ def block(pkg: str, tree_members: list[str] | None) -> str: return "\n".join(lines) + "\n" +def _str_list(items: list[str], indent: str) -> list[str]: + """A list literal as buildifier prints it: one element inline, more one + per line (the form gazelle leaves alone, so bazel-sync is a fixed point).""" + if len(items) == 1: + return [f'["{items[0]}"]'] + return ["["] + [f'{indent} "{i}",' for i in items] + [f"{indent}]"] + + +def _partition_glob(include: tuple[str, ...], exclude: list[str], allow_empty: bool) -> list[str]: + """`glob(...)` lines for a filegroup's srcs (8-space argument indent).""" + inc = _str_list(list(include), " ") + lines = [" srcs = glob(", " " + inc[0]] + inc[1:] + lines[-1] += "," + if allow_empty: + lines.append(" allow_empty = True,") + if exclude: + exc = _str_list(sorted(exclude), " ") + lines.append(" exclude = " + exc[0]) + lines += exc[1:] + lines[-1] += "," + return lines + + def repo_files_block(pkg: str, packages: list[str]) -> str: """The repo_files block of pkg ("." is the root, which aggregates).""" root = pkg == "." - exclude = REPO_FILES_EXCLUDE + (ROOT_REPO_FILES_EXCLUDE if root else ()) - lines = [ - REPO_BEGIN, - "", - "filegroup(", - ' name = "repo_files",', - " srcs = glob(", - ' ["**"],', - " exclude = [", - ] - lines += [f' "{e}",' for e in sorted(exclude)] - # No allow_empty: every package holds at least its BUILD.bazel. - lines += [" ],"] + lines = [REPO_BEGIN] + debris = list(REPO_FILES_EXCLUDE) + (list(ROOT_REPO_FILES_EXCLUDE) if root else []) + targets = [] + for name, include, own_exclude in REPO_PARTITIONS: + if name == OTHER_PARTITION: + exclude = debris + list(own_exclude) + else: + exclude = ["**/node_modules/**"] + (list(ROOT_REPO_FILES_EXCLUDE) if root else []) + list(own_exclude) + # repo_other_files always holds the package's BUILD.bazel. + body = _partition_glob(include, exclude, allow_empty=name != OTHER_PARTITION) + if root: + body.append(" ) + [") + body += [f' "//{p}:{name}",' for p in packages if p != "."] + body.append(" ],") + else: + body.append(" ),") + targets.append((name, body + [f' visibility = ["{REPO_PARTITION_VISIBILITY}"],'])) if root: - lines += [" ) + ["] - lines += [f' "//{p}:repo_files",' for p in packages if p != "."] - lines += [" ],", " visibility = ["] - lines += [f' "{v}",' for v in REPO_FILES_VISIBILITY] - lines += [" ],"] - else: - lines += [" ),", ' visibility = ["//:__pkg__"],'] - lines += [")", "", REPO_END] + union = [" srcs = ["] + [f' ":{n}",' for n, _, _ in REPO_PARTITIONS] + [" ],"] + union += [" visibility = ["] + [f' "{v}",' for v in REPO_FILES_VISIBILITY] + [" ],"] + targets.append(("repo_files", union)) + for name, body in sorted(targets): + lines += ["", "filegroup(", f' name = "{name}",'] + body + [")"] + lines += ["", REPO_END] return "\n".join(lines) + "\n" diff --git a/tools/docsmint/BUILD.bazel b/tools/docsmint/BUILD.bazel index da07e7a9b..5b10ddb51 100644 --- a/tools/docsmint/BUILD.bazel +++ b/tools/docsmint/BUILD.bazel @@ -29,12 +29,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/BUILD.bazel b/tools/nogo/BUILD.bazel index 1e44a0bca..f1e2331d6 100644 --- a/tools/nogo/BUILD.bazel +++ b/tools/nogo/BUILD.bazel @@ -43,15 +43,60 @@ go_test( ], ) +# The nested module's go.mod, for //:nested_go_modules. +exports_files( + ["go.mod"], + visibility = ["//:__pkg__"], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -60,7 +105,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/depguard/BUILD.bazel b/tools/nogo/analyzers/depguard/BUILD.bazel index 8dda85443..6b3bb4f56 100644 --- a/tools/nogo/analyzers/depguard/BUILD.bazel +++ b/tools/nogo/analyzers/depguard/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/errcheck/BUILD.bazel b/tools/nogo/analyzers/errcheck/BUILD.bazel index 2d1eaa1b2..28d7c90d7 100644 --- a/tools/nogo/analyzers/errcheck/BUILD.bazel +++ b/tools/nogo/analyzers/errcheck/BUILD.bazel @@ -26,12 +26,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -40,7 +79,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/forbidigo/BUILD.bazel b/tools/nogo/analyzers/forbidigo/BUILD.bazel index ae87048b5..a5769977d 100644 --- a/tools/nogo/analyzers/forbidigo/BUILD.bazel +++ b/tools/nogo/analyzers/forbidigo/BUILD.bazel @@ -25,12 +25,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -39,7 +78,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/gosec/BUILD.bazel b/tools/nogo/analyzers/gosec/BUILD.bazel index 0f13fb1c9..68381b0cd 100644 --- a/tools/nogo/analyzers/gosec/BUILD.bazel +++ b/tools/nogo/analyzers/gosec/BUILD.bazel @@ -29,12 +29,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/misspell/BUILD.bazel b/tools/nogo/analyzers/misspell/BUILD.bazel index 0b2053638..525f22c2c 100644 --- a/tools/nogo/analyzers/misspell/BUILD.bazel +++ b/tools/nogo/analyzers/misspell/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/sloglint/BUILD.bazel b/tools/nogo/analyzers/sloglint/BUILD.bazel index 9289cddb1..b90003388 100644 --- a/tools/nogo/analyzers/sloglint/BUILD.bazel +++ b/tools/nogo/analyzers/sloglint/BUILD.bazel @@ -26,12 +26,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -40,7 +79,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/unconvert/BUILD.bazel b/tools/nogo/analyzers/unconvert/BUILD.bazel index 2f988fd1a..7351d4a5e 100644 --- a/tools/nogo/analyzers/unconvert/BUILD.bazel +++ b/tools/nogo/analyzers/unconvert/BUILD.bazel @@ -25,12 +25,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -39,7 +78,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/unparam/BUILD.bazel b/tools/nogo/analyzers/unparam/BUILD.bazel index f4c612814..76d18f163 100644 --- a/tools/nogo/analyzers/unparam/BUILD.bazel +++ b/tools/nogo/analyzers/unparam/BUILD.bazel @@ -26,12 +26,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -40,7 +79,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/internal/analyzertest/BUILD.bazel b/tools/nogo/internal/analyzertest/BUILD.bazel index c7734cf4d..b2202ef6c 100644 --- a/tools/nogo/internal/analyzertest/BUILD.bazel +++ b/tools/nogo/internal/analyzertest/BUILD.bazel @@ -15,12 +15,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -29,7 +68,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/internal/golangci/BUILD.bazel b/tools/nogo/internal/golangci/BUILD.bazel index 835acc846..27fb884c5 100644 --- a/tools/nogo/internal/golangci/BUILD.bazel +++ b/tools/nogo/internal/golangci/BUILD.bazel @@ -37,12 +37,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -51,7 +90,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/internal/position/BUILD.bazel b/tools/nogo/internal/position/BUILD.bazel index 200493e23..8cc8a2947 100644 --- a/tools/nogo/internal/position/BUILD.bazel +++ b/tools/nogo/internal/position/BUILD.bazel @@ -11,12 +11,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -25,7 +64,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- From 5a5b7cfd66b86c176ec311a330363a9fb0bcbabb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 05:50:09 +0000 Subject: [PATCH 008/151] ci(bazel): pin the slowest cmd/bd Dolt-server tests to their own shards (#7413) * ci(bazel): pin the slowest cmd/bd Dolt-server tests to their own shards The cmd-dolt lane's wall time when its tests re-run is the bd_test race compile plus its slowest shard. rules_go shards round-robin by test index, so whichever shard drew TestSharedServerConcurrent and the TestInit* family ran 117-170s while the median shard ran ~55s. tools/bazel/go_test_pinned_shard.sh runs a go_test binary with the tests a committed manifest pins (measured >= 5s, bin-packed longest-first by tools/bazel/pin_shards.py) in shards of their own, and every other test in the binary's own round-robin shards with the pinned set skipped, so each test still runs exactly once and new tests need no manifest edit. bd_dolt_server_test goes from 16 to 32 shards (16 pinned + 16 round-robin), regression_test from 8 to 16 (10 + 6; its go_test becomes the manual regression_suite binary behind an sh_test of the old name). Pinning exposed a pre-existing order dependence: TestCLI_Import failed whenever it was the binary's first test, because runBDInProcess reused the workspace discovery an earlier in-process execution cached and `bd init` in a fresh directory resolved the previous one's .beads. The in-process runners now reset beads/git discovery caches per execution. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): pin cmd/bd Dolt-server shards from two measured runs; regression in 14 shards The bd manifest now takes each test's slowest of the 16- and 32-shard runs (pin_shards.py over both), and regression_test drops to 10 pinned + 4 round-robin shards: its unpinned tests total ~128s, so 6 round-robin shards were mostly fixed cost. Co-Authored-By: Claude Opus 5.5 (1M context) * test(scripts): place the pinned-shard tests by what they read (#7414 split) The manifest-name scan reads every cmd/bd and tests/regression _test.go file, so it moves to go_test_sources_test (with tests/regression's manifest); the wrapper, generator and wiring checks stay in scripts_test. Shared helpers live in a helpers-only file both targets compile, and repoFiles takes #7414's minimum-file count. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 13 +- cmd/bd/BUILD.bazel | 24 ++- cmd/bd/cli_fast_test.go | 15 ++ cmd/bd/dolt_server_pinned_shards.txt | 103 ++++++++++ engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 2 +- engdocs/CI_TEST_SURFACE_AUDIT.md | 2 +- engdocs/TESTING.md | 11 +- scripts/BUILD.bazel | 5 + scripts/ci_workflow_test.go | 26 ++- scripts/pinned_shards_helpers_test.go | 65 ++++++ scripts/pinned_shards_sources_test.go | 48 +++++ scripts/pinned_shards_test.go | 268 +++++++++++++++++++++++++ scripts/pr_risk_bazel_coverage_test.go | 13 +- tests/regression/BUILD.bazel | 43 +++- tests/regression/pinned_shards.txt | 55 +++++ tools/bazel/BUILD.bazel | 8 + tools/bazel/go_test_pinned_shard.sh | 90 +++++++++ tools/bazel/pin_shards.py | 117 +++++++++++ 18 files changed, 867 insertions(+), 41 deletions(-) create mode 100644 cmd/bd/dolt_server_pinned_shards.txt create mode 100644 scripts/pinned_shards_helpers_test.go create mode 100644 scripts/pinned_shards_sources_test.go create mode 100644 scripts/pinned_shards_test.go create mode 100644 tests/regression/pinned_shards.txt create mode 100755 tools/bazel/go_test_pinned_shard.sh create mode 100755 tools/bazel/pin_shards.py diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 66322c9d5..5ca224450 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -2065,12 +2065,12 @@ jobs: run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" # The cmd/bd Dolt-server tier, as --config=doltserver-cmd: the whole - # integration-tagged cmd/bd suite (//cmd/bd:bd_dolt_server_test, 16 - # shards) against a hermetic dolt sql-server, i.e. the Dolt-gated cmd/bd + # integration-tagged cmd/bd suite (//cmd/bd:bd_dolt_server_test, 32 + # shards, the slowest tests pinned to 16 of them) against a hermetic dolt sql-server, i.e. the Dolt-gated cmd/bd # tests (TestCLI_*, TestInit*, the store-backed suites) that every other # lane and legacy job skips with BEADS_TEST_SKIP=dolt or leaves out of its # manifest; plus the differential regression suite - # (//tests/regression:regression_test, 8 shards: the candidate + # (//tests/regression:regression_test, 14 shards: the candidate # bd_for_tests against the catalog-pinned v0.49.6 release, formerly # regression.yml). --config=doltserver-cmd has exactly # --config=integration's build flags, so it shares bazel-integration's @@ -2120,9 +2120,10 @@ jobs: RBE_FORK_PR: ${{ github.event.pull_request.number }} # Remote: a compile shared with bazel-integration plus the longest - # of 16 shards (~3 minutes measured; each shard pays ~30-40 s of - # TestMain and its own dolt sql-server; Go's -test.timeout=19m, - # Bazel's 1200s cap). Mode cache on a 4-CPU runner: the compile hits + # of 32 shards (~1 minute: tools/bazel/go_test_pinned_shard.sh pins + # the slowest tests to shards of their own; each shard pays a few + # seconds of TestMain and its own dolt sql-server; Go's + # -test.timeout=19m, Bazel's 1200s cap). Mode cache on a 4-CPU runner: the compile hits # the read-only cache, and a PR that changes cmd/bd or what it imports # runs every shard locally, four at a time (~2,200 test-seconds # measured unloaded, slower four-up: ~20-30 minutes); with the cache diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index f220e60a9..a38bc8bd7 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1563,10 +1563,16 @@ sh_test( # instead of skipping every Dolt-gated test into a cached pass. These are # the cmd/bd tests no other lane runs: every other configuration of bd_test # passes BEADS_TEST_SKIP=dolt (prcore/ci, integration) or runs only its -# manifest (bd_embedded_test, bd_proxied_test). No -test.run/-test.skip: the Go binary shards -# itself over every top-level test (rules_go's TEST_SHARD_INDEX), so each -# runs in exactly one shard, and check_testcases.py fails a shard that ran -# none. It repeats bd_test's `env` (the wrapper cannot inherit it). +# manifest (bd_embedded_test, bd_proxied_test). Shards 1-16 each run the +# slowest tests dolt_server_pinned_shards.txt pins to them (duration-balanced +# by tools/bazel/pin_shards.py); shards 17-32 are the Go binary's own +# round-robin shards (rules_go's TEST_SHARD_INDEX) over every other top-level +# test, a test added since the manifest was written included +# (tools/bazel/go_test_pinned_shard.sh). So each test runs in exactly one +# shard, and check_testcases.py fails a shard that ran none. Round-robin +# alone left the slowest of 16 shards at 2-3x the median (TestSharedServerConcurrent, +# the TestInit* family). It repeats bd_test's `env` (the wrapper cannot +# inherit it). # # Tags: # dolt-server-cmd: the cmd/bd Dolt-server tier; starts hermetic dolt @@ -1580,11 +1586,15 @@ sh_test( # dump) before Bazel's 1200s action limit (eternal under the lane's # --test_timeout), which kills without one. timeout = "eternal", - srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :bd_test)"], + srcs = ["//tools/bazel:go_test_pinned_shard.sh"], + args = [ + "$(rootpath :dolt_server_pinned_shards.txt)", + "$(rootpath :bd_test)", + ], data = [ ":bd_for_tests", ":bd_test", + ":dolt_server_pinned_shards.txt", ":preflight_go_fixture", "@go_sdk//:bin/gofmt", ], @@ -1607,7 +1617,7 @@ sh_test( # against the one shared server. "GOMAXPROCS": "4", }, - shard_count = 16, + shard_count = 32, tags = ["dolt-server-cmd"], ) diff --git a/cmd/bd/cli_fast_test.go b/cmd/bd/cli_fast_test.go index 47c82ced1..096a64afd 100644 --- a/cmd/bd/cli_fast_test.go +++ b/cmd/bd/cli_fast_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/testutil/bazeltest" "github.com/steveyegge/beads/internal/testutil/credentialcmd" ) @@ -151,6 +153,17 @@ func createTempDirWithCleanup(t *testing.T) string { // TestMain having installed it). func restoreEnvSnapshot(before []string) { restoreProcessEnv(before) } +// resetInProcessDiscovery drops the workspace discovery an earlier in-process +// execution cached (beads' RepoContext, git's repository context), so each +// one resolves its .beads from its own working directory as a bd process +// would. Without it TestCLI_Import, run as the binary's first test, ran +// `bd init` in a fresh directory against the .beads it had just exported +// from ("Found existing Dolt database"). +func resetInProcessDiscovery() { + beads.ResetCaches() + git.ResetCaches() +} + // runBDInProcess runs bd commands in-process by calling rootCmd.Execute // This is ~10-20x faster than exec.Command because it avoids process spawn overhead func runBDInProcess(t *testing.T, dir string, args ...string) string { @@ -171,6 +184,7 @@ func runBDInProcess(t *testing.T, dir string, args ...string) string { if err := os.Chdir(dir); err != nil { t.Fatalf("Failed to chdir to %s: %v", dir, err) } + resetInProcessDiscovery() // Capture stdout/stderr rOut, wOut, _ := os.Pipe() @@ -1235,6 +1249,7 @@ func runBDInProcessAllowError(t *testing.T, dir string, args ...string) (string, if err := os.Chdir(dir); err != nil { t.Fatalf("Failed to chdir to %s: %v", dir, err) } + resetInProcessDiscovery() rOut, wOut, _ := os.Pipe() rErr, wErr, _ := os.Pipe() diff --git a/cmd/bd/dolt_server_pinned_shards.txt b/cmd/bd/dolt_server_pinned_shards.txt new file mode 100644 index 000000000..d7ab46c5b --- /dev/null +++ b/cmd/bd/dolt_server_pinned_shards.txt @@ -0,0 +1,103 @@ +# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by +# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage). +# +# Every top-level test measured at >= 5s, bin-packed +# longest-first onto these shards; every other test runs in the +# binary's own round-robin shards. Measured seconds per pinned shard: +# 59 60 59 62 59 59 58 57 57 57 62 62 57 57 57 62 +# Unpinned tests: 3322, 570s in total. +# +# +1 TestCLI_NoteAddMisplacedSyntax +1 TestSharedServerConcurrent +2 TestAcquireInitMutationGateWaitsPastGenericBudget +2 TestCLI_ReparentDottedIDExcludesOldParent +2 TestContributorRoutingDaemon +2 TestMultiIDUpdatePartialFailureJSONReportsFailedIDs +2 TestRoutingWithAllOperationalModes +3 TestADORoundTripLossyPriority +3 TestCLI_UpdateEphemeral +3 TestDetermineAutoRoutedRepoPath_DefaultFallback +3 TestDirectTrackerConformance +3 TestGetNotionConfigReadsDBPathWhenStoreUnset +4 TestBareParentWorktreeCoreCommandsWithoutRedirect +4 TestCLI_Blocked +4 TestDoltAutoCommit_Batch_DefersCommit +4 TestE2E_DoctorFixMetadataRoundtrip +4 TestE2E_MigrateDoltMetadata +4 TestInitDatabaseFlag +5 TestADORoundTripLabels +5 TestBackwardCompatContributorConfig +5 TestContributorRoutingSyncBranch +5 TestEmbeddedChangeDirOverridesInheritedBeadsDir +5 TestImportBlockedDuringMigrationFreeze +5 TestResolveCloseTargets +6 TestADORoundTripBlockedStatus +6 TestCLI_CommentsAddReservedWordRejectedThroughCobraDispatch +6 TestCLI_UpdateEphemeralMutualExclusion +6 TestDepRoutedTargetOpensReadOnly +6 TestE2E_AutoStartedRepoLocalServerPersistsAcrossCommands +6 TestInitCommand +7 TestAutoExportSkipsWhenExistingJSONLHasIDsMissingFromStore +7 TestCLI_CreateRejectsFlagLikeTitles +7 TestContributorRoutingTracer +7 TestGlobalDBIdentityCheck +7 TestMultiIDUpdateAllGoodPathUnchanged +7 TestSharedServerDeleteIfRevisionSingleWinner +8 TestCLI_Import_GlobalModeIgnoresProjectConfigYAMLPrefix +8 TestCLI_Update +8 TestCreateDepsAtomicity +8 TestDoctorPathArgMaintenanceSkippedByTargetFreeze +8 TestOpenRoutedReadStore_ContributorRouting +8 TestPrimeBinaryPortfolio +9 TestCLI_CommentAbbreviatedIDRejectedWithTruthfulMessage +9 TestContributorRoutingDirect +9 TestDoltRemoteAddPersistsSyncRemoteToSharedWorktreeConfig +9 TestInitNonInteractiveAlwaysSetsRole +9 TestServeAnswersFromARegisteredBackendStore +9 TestSharedServerVersionBumpDoesNotPromoteCursor +10 TestAutoMigrateSkippedDuringMigrationFreeze +10 TestCLI_Import +10 TestCLI_NoteListMisplacedSyntax +10 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget +10 TestResolveAndGetIssueForMutationExactSurfacesRoutedAbbreviationRefusal +10 TestSharedServerIfRevisionSingleWinner +11 TestBEADS_DIRPrecedence +11 TestDoctorCheckHealthReportsVersionMismatchOnRepoLocalPort +11 TestDoctorPathArgBlockedByTargetFreeze +11 TestInitBackendFlag +11 TestInitContributorSetsBeadsRoleContributor +11 TestInit_WithBEADS_DIR_DoltBackend +11 TestTemplateSuite +12 TestCLI_Labels +12 TestCLI_NoteCommand +12 TestCLI_Show_SupersedesIsNotABlocker +12 TestCheckBeadGateCrossRigPrefixRoute +12 TestInitIfMissing +12 TestInitMetricsEmittedPerDoltMode +12 TestReadonlyShowPreservesLastTouched +13 TestCLI_DepRemove +13 TestCLI_UpdateAppendNotes +13 TestDepListCrossRigRouting +13 TestExplicitRepoOverride +13 TestInitIfMissingMatchingPrefixSkips +13 TestMultiIDUpdatePartialFailureExitsNonzero +14 TestCLI_CommentsAddShortID +14 TestCLI_DepTree +14 TestContributorPlanningGitIgnoresInheritedRouting +14 TestContributorRoutingLocalOnly +14 TestMaintainerRoutingUnaffected +14 TestWhereCommand_ReadsPrefixFromEmbeddedStore +15 TestADORoundTripCoreFields +15 TestBdMolPour_NeedsAndWaitsForSameTarget_SpawnerOpenBlocksWaiter +15 TestCLI_CreateDryRun +15 TestCLI_PriorityFormats +15 TestInitBEADS_DIR +15 TestInitPromptNonGitRepo +16 TestBdMolBond_ResolvesPlainFormulaName +16 TestCLI_UpdateLabels +16 TestCLI_UpdatePersistent +16 TestContributorRoutingNoDb +16 TestInitDoltMetadata +16 TestInitMetricsEnvConfigPrecedence +16 TestInitPromptExistingRole diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 09ebaa67f..a9f34e510 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1274,7 +1274,7 @@ lane runs `//internal/storage/dolt:dolt_server_full_test` (16 shards) and ### Regression Tests `regression.yml` is retired. The suite runs as -`//tests/regression:regression_test` (8 shards) in `bazel.yml`'s +`//tests/regression:regression_test` (14 shards) in `bazel.yml`'s `bazel-cmd-dolt` lane, which `pr.yml`'s `CI Gate / Required` requires (`BAZEL_CMD_DOLT_REQUIRED`), so regression is now part of the aggregate gate on PRs and merge groups instead of a separate, path-detected advisory diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index 202b0dd33..1f32213fd 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -199,7 +199,7 @@ Key jobs preserved by display name: | Workflow | Triggers | Main validation | |---|---|---| -| `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (8 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | +| `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (14 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | | `bazel.yml` `bazel-test` (formerly `cross-version-smoke.yml`) | PR, merge group, push to `main` | `//tests/upgrade_smoke:upgrade_smoke__test` for the newest 30 catalog-pinned releases, via `scripts/upgrade-smoke-test.sh` with `PREV_BIN`; cached until their inputs change. | | `bazel.yml` `bazel-test` (formerly `migration-test.yml`) | PR, merge group, push to `main` | `//tests/migration:historical_upgrade__test`, one target per reviewed release (14, v0.9.1 source-built offline) plus `legacy_bridge_test`, against catalog-pinned release binaries; cached until their inputs change. Local entrypoint stays `scripts/migration-test/run.sh`. | | `nightly.yml` | Daily schedule, manual | `go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./...` with `BEADS_TEST_SKIP=dolt`; checks coverage >= 30%. | diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 9adaed89f..41efd4955 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -40,7 +40,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Release cross-compile (`bazel-release-cross`) | `./scripts/ci/bazel-release-cross-compile.sh` | Every `go_library`/`go_binary` for each row of `scripts/ci/release-targets.txt`, cgo off, with nogo. | | Integration (`bazel-integration`) | `bazel test //... --config=integration` | The `integration`-tagged build. Runs with the read-only cache too. | | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | -| cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 16 shards of the integration-tagged cmd/bd suite. | +| cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | | Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 30 shards, each with a Dolt server. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | @@ -193,8 +193,13 @@ advisory `bazel-cmd-dolt` job) runs the whole integration-tagged cmd/bd suite on the `local` backend: the Dolt-gated cmd/bd tests (`TestCLI_*`, the init and store-backed suites) that every other lane skips with `BEADS_TEST_SKIP=dolt` or leaves out of its manifest. It shares -`--config=integration`'s build, passes the binary no test selection (the Go -binary shards itself over every top-level test, 16 shards), and runs where +`--config=integration`'s build and runs it in 32 shards: 16 run the slowest +tests `cmd/bd/dolt_server_pinned_shards.txt` pins to them, and 16 are the Go +binary's own round-robin shards over every other top-level test +(`tools/bazel/go_test_pinned_shard.sh`; `//tests/regression:regression_test` +splits the same way). After a change that moves test durations, regenerate the +manifest from a run of the lane with `tools/bazel/pin_shards.py` (see its +usage); a stale manifest only costs balance, never coverage. It runs where the integration lane runs (remote, or with the read-only cache). pr.yml's gate requires it once `BAZEL_CMD_DOLT_REQUIRED` is `"true"`; pr.yml then also passes bazel.yml `cmd-dolt-required: true`, and the PR's diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 92a065548..ef89f2709 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -53,6 +53,8 @@ go_test( "makefile_bazel_primary_test.go", "migration_corpus_bazel_test.go", "nogo_lint_policy_test.go", + "pinned_shards_helpers_test.go", + "pinned_shards_test.go", "pr_lanes_bazel_coverage_test.go", "pr_risk_bazel_coverage_test.go", "rbe_worker_env_test.go", @@ -131,6 +133,8 @@ go_test( "embedded_shard_count_test.go", "host_tools_test.go", "managed_local_lane_test.go", + "pinned_shards_helpers_test.go", + "pinned_shards_sources_test.go", "repo_helpers_test.go", "scripts_targets_test.go", "shard_manifest_test.go", @@ -157,6 +161,7 @@ go_test( "//cmd/bd:repo_other_files", "//internal/storage/dolt:repo_other_files", "//internal/storage/embeddeddolt:repo_other_files", + "//tests/regression:repo_other_files", "//tools/bazel:check_shard_coverage.py", "//tools/bazel:equivalence_allowlist.txt", ], diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index eb75ab320..174e8255e 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -5801,9 +5801,11 @@ func btoi(b bool) int { // whole integration-tagged cmd/bd suite against a hermetic dolt sql-server, // because every other lane, Bazel or go test, runs cmd/bd with // BEADS_TEST_SKIP=dolt or only a manifest's tests, so its Dolt-gated tests -// ran nowhere. Coverage is exact by construction: the target passes the -// binary no test selection, the Go binary shards itself over every -// top-level test, and check_testcases.py fails a shard that ran none. +// ran nowhere. Coverage is exact by construction: the target's only test +// selection is go_test_pinned_shard.sh's, whose round-robin shards skip +// exactly the tests its pinned shards run (TestPinnedShardWrapperSplit, +// TestPinnedShardManifests), and check_testcases.py fails a shard that ran +// none. func TestBazelCmdDoltJob(t *testing.T) { const config, target = "doltserver-cmd", "bd_dolt_server_test" workflow := readCIWorkflow(t, bazelWorkflowName) @@ -5881,8 +5883,8 @@ func TestBazelCmdDoltJob(t *testing.T) { t.Error(err) } for _, want := range []string{ - `srcs = ["//tools/bazel:go_test_variant.sh"]`, - `args = ["$(rootpath :bd_test)"]`, + `srcs = ["//tools/bazel:go_test_pinned_shard.sh"]`, + "args = [\n \"$(rootpath :dolt_server_pinned_shards.txt)\",\n \"$(rootpath :bd_test)\",\n ]", `tags = ["dolt-server-cmd"]`, `"GOMAXPROCS": "4"`, `"BEADS_TEST_GIT_IDENTITY": "1"`, @@ -5891,12 +5893,14 @@ func TestBazelCmdDoltJob(t *testing.T) { t.Errorf("%s lacks %s", target, want) } } - // 16 shards: each pays ~30-40 s of fixed cost (TestMain, its own dolt - // sql-server), so more shards mostly add farm actions; with the suite's - // long pole gone the slowest of 16 stays within bazel-integration's - // critical path. Change it together with bazel.yml's comment. - if m := regexp.MustCompile(`shard_count = (\d+)`).FindStringSubmatch(rule); m == nil || m[1] != "16" { - t.Errorf("%s shard_count = %v, want 16", target, m) + // 32 shards: 16 that run the slowest tests the manifest pins to them + // and 16 round-robin shards over the rest. Measured, a shard's fixed + // cost (TestMain, its own dolt sql-server) is a few seconds; the wall + // time is the serial sum of its tests, which round-robin alone left at + // 2-3x the median on the shard that drew the long tests. Change it + // together with bazel.yml's comment. + if m := regexp.MustCompile(`shard_count = (\d+)`).FindStringSubmatch(rule); m == nil || m[1] != "32" { + t.Errorf("%s shard_count = %v, want 32", target, m) } envKey := regexp.MustCompile(`"(BEADS_TEST_[A-Z_]+)": "\$\(rlocationpath [^)]+\)"`) for _, m := range envKey.FindAllStringSubmatch(bdTest, -1) { diff --git a/scripts/pinned_shards_helpers_test.go b/scripts/pinned_shards_helpers_test.go new file mode 100644 index 000000000..862fcd0b4 --- /dev/null +++ b/scripts/pinned_shards_helpers_test.go @@ -0,0 +1,65 @@ +package scripts_test + +// Helpers for the pinned-shard tests, shared by scripts_test +// (pinned_shards_test.go) and go_test_sources_test +// (pinned_shards_sources_test.go). + +import ( + "regexp" + "strconv" + "strings" + "testing" +) + +// pinnedShardWrapper is the sh_test src that pins slow tests to shards. +const pinnedShardWrapper = "tools/bazel/go_test_pinned_shard.sh" + +// pinnedShardTargets are the sh_tests that run a go_test binary through +// tools/bazel/go_test_pinned_shard.sh: the BUILD file, the rule, its +// manifest, and the directory holding the go_test's sources. +var pinnedShardTargets = []struct { + build, rule, manifest, pkg string +}{ + {"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd"}, + {"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression"}, +} + +var pinnedLineRe = regexp.MustCompile(`^([1-9][0-9]*) (Test[A-Za-z0-9_]*)$`) + +// readPinnedShardManifest parses a pinned-shard manifest the way +// go_test_pinned_shard.sh does, reporting malformed lines, duplicate names +// and empty shards, and returns the pinned names and the pinned shard count. +func readPinnedShardManifest(t *testing.T, root, manifest string) (map[string]int, int) { + t.Helper() + pinned := map[string]int{} + shards := map[int]int{} + maxShard := 0 + for i, line := range strings.Split(readPolicyFile(t, root, manifest), "\n") { + line, _, _ = strings.Cut(line, "#") + line = strings.TrimSpace(line) + if line == "" { + continue + } + m := pinnedLineRe.FindStringSubmatch(line) + if m == nil { + t.Errorf("%s:%d: %q is not \" \"", manifest, i+1, line) + continue + } + shard, _ := strconv.Atoi(m[1]) + if _, dup := pinned[m[2]]; dup { + t.Errorf("%s: %s pinned twice", manifest, m[2]) + } + pinned[m[2]] = shard + shards[shard]++ + maxShard = max(maxShard, shard) + } + for s := 1; s <= maxShard; s++ { + if shards[s] == 0 { + t.Errorf("%s: pinned shard %d is empty", manifest, s) + } + } + if maxShard == 0 { + t.Errorf("%s pins nothing", manifest) + } + return pinned, maxShard +} diff --git a/scripts/pinned_shards_sources_test.go b/scripts/pinned_shards_sources_test.go new file mode 100644 index 000000000..fea3b49e0 --- /dev/null +++ b/scripts/pinned_shards_sources_test.go @@ -0,0 +1,48 @@ +package scripts_test + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "testing" +) + +var goTopLevelTestRe = regexp.MustCompile(`(?m)^func (Test[A-Za-z0-9_]*)\(t \*testing\.T\) \{`) + +// Every name a pinned-shard manifest pins is a top-level test of the +// target's package: a stale name runs nowhere, and a renamed test silently +// falls back to the round-robin shards (regenerate with +// tools/bazel/pin_shards.py). +func TestPinnedShardManifestNamesAreTests(t *testing.T) { + root := sourceRepoRoot(t) + for _, tgt := range pinnedShardTargets { + t.Run(tgt.rule, func(t *testing.T) { + tests := map[string]bool{} + srcs, err := filepath.Glob(filepath.Join(root, tgt.pkg, "*_test.go")) + if err != nil || len(srcs) == 0 { + t.Fatalf("no *_test.go in %s: %v", tgt.pkg, err) + } + for _, src := range srcs { + b, err := os.ReadFile(src) + if err != nil { + t.Fatal(err) + } + for _, m := range goTopLevelTestRe.FindAllStringSubmatch(string(b), -1) { + tests[m[1]] = true + } + } + pinned, _ := readPinnedShardManifest(t, root, tgt.manifest) + var stale []string + for name := range pinned { + if !tests[name] { + stale = append(stale, name) + } + } + sort.Strings(stale) + for _, name := range stale { + t.Errorf("%s pins %s, which is not a top-level test in %s/*_test.go; regenerate it (tools/bazel/pin_shards.py)", tgt.manifest, name, tgt.pkg) + } + }) + } +} diff --git a/scripts/pinned_shards_test.go b/scripts/pinned_shards_test.go new file mode 100644 index 000000000..ff95e9f3f --- /dev/null +++ b/scripts/pinned_shards_test.go @@ -0,0 +1,268 @@ +package scripts_test + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// fakeTestBinary writes a stand-in for a go_test binary that prints its +// arguments and the sharding environment it was started with. +func fakeTestBinary(t *testing.T, dir string) { + t.Helper() + script := `#!/usr/bin/env bash +echo "args=$*" +echo "total=${TEST_TOTAL_SHARDS-unset} index=${TEST_SHARD_INDEX-unset} status=${TEST_SHARD_STATUS_FILE-unset} fromBazel=${GO_TEST_RUN_FROM_BAZEL-unset}" +` + if err := os.WriteFile(filepath.Join(dir, "bin"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } +} + +func runPinnedShard(t *testing.T, dir, manifest string, env ...string) (string, error) { + t.Helper() + wrapper := filepath.Join(sourceRepoRoot(t), "tools", "bazel", "go_test_pinned_shard.sh") + if err := os.WriteFile(filepath.Join(dir, "manifest.txt"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command(requireHostTool(t, "bash"), wrapper, "manifest.txt", "bin", "-test.parallel=4") + cmd.Dir = dir + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH")}, env...) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// The wrapper's split: pinned shards run exactly their tests with the +// binary's own sharding off; the rest are the binary's round-robin shards +// minus every pinned test; unsharded runs everything. Every test runs in +// exactly one shard because the skip set is the union of the run sets. +func TestPinnedShardWrapperSplit(t *testing.T) { + const manifest = `# comment +1 TestSlow # trailing comment +2 TestB +2 TestA + +` + status := filepath.Join(t.TempDir(), "status") + cases := []struct { + name string + env []string + want []string + }{ + {"unsharded", nil, []string{ + "args=-test.parallel=4\n", + "total=unset index=unset status=unset fromBazel=1\n", + }}, + {"pinned shard 1", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=0", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.run=^(TestSlow)$ -test.parallel=4\n", + "total=unset index=unset status=unset fromBazel=1\n", + }}, + {"pinned shard 2", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=1", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.run=^(TestB|TestA)$ -test.parallel=4\n", + "total=unset index=unset", + }}, + {"first round-robin shard", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=2", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.skip=^(TestSlow|TestB|TestA)$ -test.parallel=4\n", + "total=3 index=0 status=" + status + " fromBazel=1\n", + }}, + {"last round-robin shard", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=4", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.skip=^(TestSlow|TestB|TestA)$ -test.parallel=4\n", + "total=3 index=2 ", + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + fakeTestBinary(t, dir) + _ = os.Remove(status) + out, err := runPinnedShard(t, dir, manifest, tc.env...) + if err != nil { + t.Fatalf("wrapper failed: %v\n%s", err, out) + } + for _, w := range tc.want { + if !strings.Contains(out, w) { + t.Errorf("output lacks %q:\n%s", w, out) + } + } + if len(tc.env) > 0 { + if _, err := os.Stat(status); err != nil { + t.Errorf("TEST_SHARD_STATUS_FILE not touched: %v", err) + } + } + }) + } +} + +func TestPinnedShardWrapperRejectsBadManifests(t *testing.T) { + cases := map[string]string{ + "gap in shard numbers": "1 TestA\n3 TestB\n", + "test pinned twice": "1 TestA\n2 TestA\n", + "not a test name": "1 BenchmarkA\n", + "regex in name": "1 TestA.*\n", + "extra field": "1 TestA TestB\n", + "shard zero": "0 TestA\n", + "no pinned shards": "# empty\n", + "no round-robin shards left": "1 TestA\n2 TestB\n3 TestC\n4 TestD\n5 TestE\n", + "more pinned than Bazel runs": "1 TestA\n2 TestB\n3 TestC\n4 TestD\n5 TestE\n6 TestF\n", + } + for name, manifest := range cases { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + fakeTestBinary(t, dir) + out, err := runPinnedShard(t, dir, manifest, "TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=0") + if err == nil { + t.Fatalf("wrapper accepted manifest %q:\n%s", manifest, out) + } + if strings.Contains(out, "args=") { + t.Errorf("wrapper ran the binary despite a bad manifest:\n%s", out) + } + }) + } +} + +// The committed manifests are well formed and each target is wired to its +// manifest with round-robin shards left over. (That every pinned name is a +// test of the package: TestPinnedShardManifestNamesAreTests.) +func TestPinnedShardManifests(t *testing.T) { + root := sourceRepoRoot(t) + for _, tgt := range pinnedShardTargets { + t.Run(tgt.rule, func(t *testing.T) { + _, maxShard := readPinnedShardManifest(t, root, tgt.manifest) + rule := bazelRuleBlock(readPolicyFile(t, root, tgt.build), tgt.rule) + if rule == "" { + t.Fatalf("%s has no rule %s", tgt.build, tgt.rule) + } + base := filepath.Base(tgt.manifest) + for _, want := range []string{ + `srcs = ["//tools/bazel:go_test_pinned_shard.sh"]`, + `"$(rootpath :` + base + `)",`, + } { + if !strings.Contains(rule, want) { + t.Errorf("%s lacks %s", tgt.rule, want) + } + } + m := shardCountPattern.FindStringSubmatch(rule) + if m == nil { + t.Fatalf("%s has no shard_count", tgt.rule) + } + if n, _ := strconv.Atoi(m[1]); n <= maxShard { + t.Errorf("%s shard_count = %d, but %s pins %d shards; round-robin shards must remain for the other tests", tgt.rule, n, tgt.manifest, maxShard) + } + }) + } +} + +// pin_shards.py bin-packs every test at or above the threshold longest +// first, keeps the round-robin tests out, and takes the slowest of several +// runs. +func TestPinShardsGenerator(t *testing.T) { + root := sourceRepoRoot(t) + python := requireHostTool(t, "python3") + logs := t.TempDir() + write := func(run, shard, body string) { + dir := filepath.Join(logs, run, shard) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + xml := `` + body + `` + if err := os.WriteFile(filepath.Join(dir, "test.xml"), []byte(xml), 0o644); err != nil { + t.Fatal(err) + } + } + write("a", "shard_1_of_2", ``) + write("a", "shard_2_of_2", ``) + write("b", "shard_1_of_1", ``) + + out := filepath.Join(t.TempDir(), "pinned.txt") + cmd := exec.Command(python, filepath.Join(root, "tools", "bazel", "pin_shards.py"), + "--shards", "2", "--min-seconds", "5", "--write", out, filepath.Join(logs, "a"), filepath.Join(logs, "b")) + if b, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("pin_shards.py: %v\n%s", err, b) + } + b, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + var got []string + for _, line := range strings.Split(string(b), "\n") { + if line != "" && !strings.HasPrefix(line, "#") { + got = append(got, line) + } + } + // LPT: A(50)->1, B(30)->2, D(28, the slower run)->2, C(25)->1. + want := []string{"1 TestA", "1 TestC", "2 TestB", "2 TestD"} + if strings.Join(got, ";") != strings.Join(want, ";") { + t.Errorf("manifest = %q, want %q", got, want) + } + if !strings.Contains(string(b), "# 75 58\n") { + t.Errorf("manifest header lacks per-shard totals 75 58:\n%s", b) + } + + cmd = exec.Command(python, filepath.Join(root, "tools", "bazel", "pin_shards.py"), + "--shards", "5", "--min-seconds", "5", filepath.Join(logs, "a")) + if b, err := cmd.CombinedOutput(); err == nil { + t.Errorf("pin_shards.py accepted 5 shards for 4 slow tests:\n%s", b) + } +} + +// pinnedShardWrapperUsers returns an error for every BUILD rule that runs +// through go_test_pinned_shard.sh other than pinnedShardTargets, and for any +// of those tagged for another lane than the cmd/bd Dolt-server tier: the +// wrapper selects and skips tests, which the retired tiers' lanes forbid +// (TestBazelRetiredLanesCannotBeNarrowed). +func pinnedShardWrapperUsers(t *testing.T, root string) []string { + t.Helper() + allowed := map[string]bool{} + for _, tgt := range pinnedShardTargets { + allowed[tgt.build+":"+tgt.rule] = true + } + const label = "//tools/bazel:go_test_pinned_shard.sh" + var errs []string + seen := 0 + for _, f := range repoFiles(t, root, 300) { + if filepath.Base(f) != "BUILD.bazel" { + continue + } + build := readPolicyFile(t, root, f) + if !strings.Contains(build, "go_test_pinned_shard.sh") { + continue + } + for _, rule := range bazelTopRules(stripStarlarkComments(build)) { + if !strings.Contains(rule, "go_test_pinned_shard.sh") { + continue + } + name := "" + if m := bazelRuleNameRe.FindStringSubmatch(rule); m != nil { + name = m[1] + } + if !strings.Contains(rule, `srcs = ["`+label+`"]`) { + if f == "tools/bazel/BUILD.bazel" && strings.HasPrefix(rule, "exports_files(") { + continue + } + errs = append(errs, f+": rule "+name+" references go_test_pinned_shard.sh other than as its sh_test src "+label) + continue + } + seen++ + if !allowed[f+":"+name] { + errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+"; add it to pinnedShardTargets after reviewing that it is not a retired tier's lane") + } + if !strings.Contains(rule, `tags = ["dolt-server-cmd"]`) { + errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+" but is not tagged exactly dolt-server-cmd") + } + } + } + if seen != len(pinnedShardTargets) { + errs = append(errs, fmt.Sprintf("found %d rules running through %s, want the %d pinnedShardTargets", seen, pinnedShardWrapper, len(pinnedShardTargets))) + } + return errs +} + +func TestPinnedShardWrapperUsers(t *testing.T) { + for _, e := range pinnedShardWrapperUsers(t, sourceRepoRoot(t)) { + t.Error(e) + } +} diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index b058e806d..a5853126f 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -759,12 +759,23 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { if err != nil { t.Fatal(err) } + // go_test_pinned_shard.sh selects and skips by design (each test in + // exactly one shard: TestPinnedShardWrapperSplit), and is reviewed + // here for the dolt-server-cmd targets only, which are none of the + // retired tiers' lanes; pinnedShardWrapperUsers fails if anything + // else runs through it. + exempt := filepath.ToSlash(rel) == pinnedShardWrapper + if exempt { + for _, e := range pinnedShardWrapperUsers(t, root) { + t.Error(e) + } + } for i, line := range strings.Split(string(data), "\n") { code := strings.TrimSpace(line) if strings.HasPrefix(code, "#") { continue } - if scriptNarrow.MatchString(code) { + if scriptNarrow.MatchString(code) && !exempt { t.Errorf("%s:%d %q can select, skip or re-run the retired tiers' lanes' tests", rel, i+1, code) } for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(code, -1) { diff --git a/tests/regression/BUILD.bazel b/tests/regression/BUILD.bazel index b0c7f12ca..2c1f527f2 100644 --- a/tests/regression/BUILD.bazel +++ b/tests/regression/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") # The differential regression suite (formerly regression.yml, which ran @@ -10,26 +11,50 @@ load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") # `go build -tags gms_pure_go ./cmd/bd`. Both bd's talk to a hermetic dolt # sql-server TestMain starts from the pinned dolt (BEADS_TEST_DOLT_SERVER=local) # where the job pulled the dolt-sql-server image; a server that cannot start -# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1). The Go binary shards -# itself over the top-level tests, so each runs in exactly one shard. +# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1). # +# The go_test is only the binary (manual: never run on its own); +# :regression_test runs it in 14 shards. Shards 1-10 each run the scenarios +# pinned_shards.txt pins to them (duration-balanced by +# tools/bazel/pin_shards.py); shards 11-14 are the Go binary's own round-robin +# shards over every other top-level test, a test added since the manifest was +# written included (tools/bazel/go_test_pinned_shard.sh), so each runs in +# exactly one shard. Round-robin alone over 8 shards left the slowest at twice +# the median (TestListAllOverridesLimitWithLabel, TestDepTypeExportFidelity). +go_test( + name = "regression_suite", + srcs = [ + "regression_test.go", + "scenarios_test.go", + ], + tags = ["manual"], + target_compatible_with = LINUX_AMD64, + deps = [ + "//internal/testutil", + "//internal/testutil/bazeltest", + ], +) + # Tags: # dolt-server-cmd: starts a hermetic dolt sql-server and needs the # integration build tag; selected by --config=doltserver-cmd (bazel.yml's # bazel-cmd-dolt lane) and excluded from every other lane, whose # configurations either drop both test files or set BEADS_TEST_SKIP=dolt. -go_test( +sh_test( name = "regression_test", size = "large", # --config=doltserver-cmd's -test.timeout=19m fires (with a goroutine # dump) before Bazel's 1200s action limit. timeout = "eternal", - srcs = [ - "regression_test.go", - "scenarios_test.go", + srcs = ["//tools/bazel:go_test_pinned_shard.sh"], + args = [ + "$(rootpath :pinned_shards.txt)", + "$(rootpath :regression_suite)", ], data = [ "BASELINE_VERSION", + "pinned_shards.txt", + ":regression_suite", "//cmd/bd:bd_for_tests", "@bd_releases//:v0.49.6", ], @@ -39,13 +64,9 @@ go_test( "BEADS_TEST_DOLT_SERVER": "local", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", }, - shard_count = 8, + shard_count = 14, tags = ["dolt-server-cmd"], target_compatible_with = LINUX_AMD64, - deps = [ - "//internal/testutil", - "//internal/testutil/bazeltest", - ], ) # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/tests/regression/pinned_shards.txt b/tests/regression/pinned_shards.txt new file mode 100644 index 000000000..0734b701a --- /dev/null +++ b/tests/regression/pinned_shards.txt @@ -0,0 +1,55 @@ +# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by +# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage). +# +# Every top-level test measured at >= 5s, bin-packed +# longest-first onto these shards; every other test runs in the +# binary's own round-robin shards. Measured seconds per pinned shard: +# 50 43 43 42 42 40 40 40 39 44 +# Unpinned tests: 40, 128s in total. +# +# +1 TestListAllOverridesLimitWithLabel +2 TestDepRemovalUnblocks +2 TestDepTypeExportFidelity +2 TestReparenting +3 TestCloseGuardNonBlocksDepTypes +3 TestDiamondDependency +3 TestListResolvedBlockerAnnotation +3 TestListStatusFilterParity +3 TestSpecialCharactersInFields +4 TestCustomTypeRoundTrip +4 TestDeleteWithDependencies +4 TestEpicWithChildrenStatus +4 TestMultipleDepTypesOnSameIssue +4 TestReopenPreservesDeps +5 TestBlocksDepCloseUnblocks +5 TestExportImportEpicTree +5 TestNonBlocksDepCloseSemantics +5 TestRemoveParent +5 TestUpdateDoesNotClobberRelationalData +6 TestMultipleCommentsRoundTrip +6 TestMultipleLabelsStressRoundTrip +6 TestParentChildDepExportBothDirections +6 TestParentChildDependency +6 TestReadySemantics +7 TestBlockedCommandParity +7 TestCountCommandParity +7 TestDeleteCascade +7 TestDepTreeOutput +7 TestTransitiveBlockingChain +8 TestEpicAllChildrenClosedNotBlocked +8 TestLargeExportImportRoundTrip +8 TestListByLabelFilter +8 TestRapidCommentAddStability +8 TestRapidLabelAddRemoveStability +9 TestBlockedEpicChildrenNotReady +9 TestChildrenCommand +9 TestCommentOnClosedIssue +9 TestDependenciesRoundTrip +9 TestQueryDSLParity +10 TestBasicLifecycle +10 TestFilteredExport +10 TestListCountParity +10 TestRapidDepAddRemoveStability +10 TestSearchByTitleSubstring +10 TestShowJSONFieldCompleteness diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 6b17dde5c..fcddec75c 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -38,6 +38,14 @@ exports_files( visibility = ["//visibility:public"], ) +# sh_test src that runs one Bazel shard of an existing go_test binary with +# its slowest tests pinned to shards of their own by a manifest (see the +# script header and pin_shards.py), e.g. //cmd/bd:bd_dolt_server_test. +exports_files( + ["go_test_pinned_shard.sh"], + visibility = ["//visibility:public"], +) + # sh_test src that runs an existing go_test binary with no network but # loopback (see the script header), e.g. //cmd/bd:bd_managed_local_test. exports_files( diff --git a/tools/bazel/go_test_pinned_shard.sh b/tools/bazel/go_test_pinned_shard.sh new file mode 100755 index 000000000..d1c2b5d7d --- /dev/null +++ b/tools/bazel/go_test_pinned_shard.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Runs one Bazel shard of an existing go_test binary with the suite's slowest +# top-level tests pinned to shards of their own, so no shard's wall time is +# set by where the binary's round-robin split happens to drop its long tests. +# Like go_test_variant.sh it reuses the go_test's binary instead of compiling +# the package again. +# +# Usage: go_test_pinned_shard.sh [binary args...] +# , : runfiles paths ($(rootpath ...)), relative to the +# workspace runfiles directory, which is the test's working directory. +# The remaining arguments (and any --test_arg) go to the binary unchanged, +# after this wrapper's own -test.run/-test.skip. +# +# The manifest (tools/bazel/pin_shards.py writes it from measured test.xml +# durations) has one " " line per pinned test, pinned +# shards numbered 1..P with none empty; '#' starts a comment. With Bazel's +# shard_count N > P: +# Bazel shards 0..P-1 run exactly their pinned tests (-test.run), with the +# binary's own sharding switched off; +# Bazel shards P..N-1 are the binary's own round-robin shards 0..N-P-1 over +# every top-level test, minus every pinned one +# (-test.skip). +# So every test runs in exactly one shard: a pinned test only in its pinned +# shard, any other test (one added since the manifest was written included) +# where the binary's round-robin puts it. A manifest name with no test runs +# nothing (scripts/pinned_shards_test.go rejects one). Unsharded (no +# TEST_TOTAL_SHARDS, e.g. --test_sharding_strategy=disabled) the binary runs +# every test. +# +# rules_go sets GO_TEST_RUN_FROM_BAZEL=1 only for the go_test itself; this +# exports it, as go_test_variant.sh does, so the binary still changes into its +# package directory. The go_test's `env` is not inherited; the target repeats +# it. +set -euo pipefail +manifest="$1" +bin="$2" +shift 2 +export GO_TEST_RUN_FROM_BAZEL=1 + +total="${TEST_TOTAL_SHARDS:-1}" +if [[ "$total" -le 1 ]]; then + exec "./$bin" "$@" +fi +index="${TEST_SHARD_INDEX:?TEST_TOTAL_SHARDS is set but TEST_SHARD_INDEX is not}" + +declare -A shard_of=() +declare -a pinned_by_shard=() +pinned_shards=0 +all="" +lineno=0 +while IFS= read -r line || [[ -n "$line" ]]; do + lineno=$((lineno + 1)) + line="${line%%#*}" + read -r shard name extra <<<"$line" || true + [[ -z "${shard:-}" ]] && continue + if ! [[ "$shard" =~ ^[1-9][0-9]*$ && "${name:-}" =~ ^Test[A-Za-z0-9_]*$ && -z "${extra:-}" ]]; then + echo "$manifest:$lineno: want \" \", got: $line" >&2 + exit 1 + fi + if [[ -n "${shard_of[$name]:-}" ]]; then + echo "$manifest:$lineno: $name is pinned twice" >&2 + exit 1 + fi + shard_of[$name]="$shard" + pinned_by_shard[shard]="${pinned_by_shard[shard]:+${pinned_by_shard[shard]}|}$name" + all="${all:+$all|}$name" + ((shard > pinned_shards)) && pinned_shards="$shard" +done <"$manifest" + +for ((s = 1; s <= pinned_shards; s++)); do + if [[ -z "${pinned_by_shard[s]:-}" ]]; then + echo "$manifest: pinned shard $s has no tests (shards must be numbered 1..P with none empty)" >&2 + exit 1 + fi +done +if ((pinned_shards == 0 || total <= pinned_shards)); then + echo "$manifest pins $pinned_shards shards; the target's shard_count ($total) must be larger" >&2 + exit 1 +fi + +if [[ -n "${TEST_SHARD_STATUS_FILE:-}" ]]; then + touch "$TEST_SHARD_STATUS_FILE" +fi +if ((index < pinned_shards)); then + unset TEST_TOTAL_SHARDS TEST_SHARD_INDEX TEST_SHARD_STATUS_FILE + exec "./$bin" "-test.run=^(${pinned_by_shard[index + 1]})\$" "$@" +fi +export TEST_TOTAL_SHARDS=$((total - pinned_shards)) +export TEST_SHARD_INDEX=$((index - pinned_shards)) +exec "./$bin" "-test.skip=^(${all})\$" "$@" diff --git a/tools/bazel/pin_shards.py b/tools/bazel/pin_shards.py new file mode 100755 index 000000000..59e58b73d --- /dev/null +++ b/tools/bazel/pin_shards.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Write a pinned-shard manifest for tools/bazel/go_test_pinned_shard.sh. + +Usage: pin_shards.py --shards P --min-seconds S [--write MANIFEST] TESTLOGS... + +TESTLOGS are a sharded test's bazel-testlogs directories (each holding +shard_*_of_*/test.xml, or one test.xml), from one or more runs. Every +top-level (a name without "/") at or above S seconds in any run is +pinned: its duration is the maximum seen, and the tests are bin-packed +longest-processing-time first onto P shards. Every other test stays in the +binary's own round-robin shards. Without --write the manifest is printed. + +The durations come from the test.xml -test.v writes (GO_TEST_WRAP_TESTV=1, as +--config=doltserver-cmd sets), so regenerate from a run of the lane itself: + + bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd + python3 tools/bazel/pin_shards.py --shards 16 --min-seconds 5 \\ + --write cmd/bd/dolt_server_pinned_shards.txt \\ + bazel-testlogs/cmd/bd/bd_dolt_server_test + +Exit status: 0 on success, 1 on bad input (no test.xml, fewer pinned tests +than shards). +""" + +import argparse +import glob +import os +import sys +import xml.etree.ElementTree as ET + + +def durations(testlogs_dirs): + """Return {top-level test name: max seconds} over every test.xml.""" + out = {} + for d in testlogs_dirs: + paths = sorted(glob.glob(os.path.join(d, "shard_*_of_*", "test.xml"))) + if not paths and os.path.exists(os.path.join(d, "test.xml")): + paths = [os.path.join(d, "test.xml")] + if not paths: + raise ValueError(f"{d}: no test.xml") + for p in paths: + for tc in ET.parse(p).getroot().iter("testcase"): + name = tc.get("name") or "" + if not name.startswith("Test") or "/" in name: + continue + secs = float(tc.get("time") or 0) + out[name] = max(out.get(name, 0.0), secs) + return out + + +def pin(durs, shards, min_seconds): + """Bin-pack the tests >= min_seconds onto shards (LPT). Returns a list of + (shard number 1..shards, [(name, secs)...]) and the per-shard totals.""" + heavy = sorted( + ((n, s) for n, s in durs.items() if s >= min_seconds), + key=lambda ns: (-ns[1], ns[0]), + ) + if len(heavy) < shards: + raise ValueError(f"{len(heavy)} tests take >= {min_seconds}s, fewer than {shards} shards") + bins = [[] for _ in range(shards)] + totals = [0.0] * shards + for name, secs in heavy: + k = min(range(shards), key=lambda i: (totals[i], i)) + bins[k].append((name, secs)) + totals[k] += secs + return bins, totals + + +def render(bins, totals, min_seconds, rest): + lines = [ + "# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by", + "# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage).", + "#", + f"# Every top-level test measured at >= {min_seconds:g}s, bin-packed", + "# longest-first onto these shards; every other test runs in the", + "# binary's own round-robin shards. Measured seconds per pinned shard:", + "# " + " ".join(f"{t:.0f}" for t in totals), + f"# Unpinned tests: {rest[0]}, {rest[1]:.0f}s in total.", + "#", + "# ", + ] + for k, tests in enumerate(bins, start=1): + for name, _ in sorted(tests): + lines.append(f"{k} {name}") + return "\n".join(lines) + "\n" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--shards", type=int, required=True) + ap.add_argument("--min-seconds", type=float, required=True) + ap.add_argument("--write") + ap.add_argument("testlogs", nargs="+") + args = ap.parse_args(argv) + if args.shards < 1: + print("--shards must be at least 1", file=sys.stderr) + return 1 + try: + durs = durations(args.testlogs) + bins, totals = pin(durs, args.shards, args.min_seconds) + except (ValueError, OSError, ET.ParseError) as e: + print(f"pin_shards.py: {e}", file=sys.stderr) + return 1 + rest = [s for s in durs.values() if s < args.min_seconds] + text = render(bins, totals, args.min_seconds, (len(rest), sum(rest))) + if args.write: + tmp = args.write + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + f.write(text) + os.replace(tmp, args.write) + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From ce582394fe33addf60c02ccd908699413ec3ca13 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 07:38:56 +0000 Subject: [PATCH 009/151] build(rbe): re-pin worker-env to gascity's go1.26.9 worker host (#7411) gastownhall/gascity bumps Go to 1.26.9 for the 2026-10-08 stdlib advisories (GO-2026-6599..6617). The shared rbe-west pool workers install the Go named in gascity main's go.mod, so once that merges they advertise a new worker-env. This is the byte-for-byte copy of gascity's tools/rbe/worker-env.txt and its pin (tools/rbe/worker-env-sync); merge it together with the gascity change or beads' remote actions stop scheduling. Co-authored-by: Claude Opus 5.5 (1M context) --- platforms/BUILD.bazel | 2 +- tools/rbe/worker-env.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 2707bcd42..66e541ea9 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -23,7 +23,7 @@ platform( name = "rbe_worker", exec_properties = { - "worker-env": "sha256:6be1381cf08e7fd66684cfce6f26aca57de2561e722c505d1f2f578e9fdeafa7", + "worker-env": "sha256:0c6c05c2196c246dba37e106a3d028e2f47f2e3d7eae0e41c2fe2bd9b2e4f6f0", }, parents = ["@bazel_tools//tools:host_platform"], visibility = ["//visibility:public"], diff --git a/tools/rbe/worker-env.txt b/tools/rbe/worker-env.txt index 17ef8ffc9..e36eb24d5 100644 --- a/tools/rbe/worker-env.txt +++ b/tools/rbe/worker-env.txt @@ -1,6 +1,6 @@ arch x86_64 dolt dolt version 2.1.8 -go go version go1.26.6 linux/amd64 +go go version go1.26.9 linux/amd64 os ubuntu 24.04 pkg bash 5.2 pkg coreutils 9.4 From 64e585e62a0fe068ba87d996e2f346a619aa242b Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 07:39:01 +0000 Subject: [PATCH 010/151] build(rbe): sync worker-env pin with gascity main (Go 1.26.9 re-pin, gastownhall/gascity#7387) (#7419) gascity #7387 (Go 1.26.9 / x/net v0.60.0 advisories) re-pinned worker-env to sha256:0c6c05c2...; rbe-west's pool workers advertise that, so every beads remote-execution run since ~06:00Z stopped in the worker-env preflight. Copied tools/rbe/worker-env.txt and the platforms/BUILD.bazel pin from gascity main; tools/rbe/worker-env-sync main now reports in step. Co-authored-by: Claude Opus 5.5 (1M context) From a13272b024a358a51fcd70c62de9e9a84cdb1f25 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 07:57:01 +0000 Subject: [PATCH 011/151] ci(bazel): serve gazelle's go_repository tools from the repository cache (#7417) * ci(bazel): serve gazelle's go_repository tools from the repository cache Every bazel.yml lane starts from a fresh output base, and before any go_deps repository can be fetched gazelle compiles fetch_repo and gazelle with `go build` in @bazel_gazelle_go_repository_tools: 18-24 s of every 2 vCPU lane's loading phase (the top ci-analytics repo_fetch entry in the embedded, proxied, cmd-dolt and test lanes). tools/bazel/go_repository_tools.bzl replaces that repository (override_repo on gazelle's non_module_deps extension). The build is reproducible, so the three binaries are pinned by sha256 per gazelle release, Go SDK and host. The rule asks the repository cache for the pinned digests first (Bazel re-hashes every hit, so a restored runner cache can only yield the pinned bytes); on a miss it builds exactly as gazelle does, inside env -i, and stores a matching build in the repository cache. An unpinned or stale combination still builds from source and prints the digests to pin. scripts/bazel_go_repository_tools_test.go keeps the pins in step with MODULE.bazel's gazelle and go_sdk versions. The runner cache key already includes MODULE.bazel, so the first push to main after this saves them. Analysis-only (--nobuild) per lane on Blacksmith 2 vCPU, main vs this change with a seeded runner cache (ga-vnycm2.8): test 54.0 -> 38.3 s, embedded 53.6 -> 31.1 s, integration 49.6 -> 35.2 s, doltserver 48.5 -> 33.0 s, proxied 46.6 -> 31.6 s, server-storage 33.5 -> 17.3 s, cmd-dolt 50.3 -> 33.0 s. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): pin gazelle's go_repository tools for Go 1.26.9 too Ready for the next go_sdk bump (gascity#7387 moves to 1.26.9 for the 2026-10-08 advisories): the same binaries keep coming from the repository cache instead of failing the pin test. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- MODULE.bazel | 16 ++ scripts/BUILD.bazel | 1 + scripts/bazel_go_repository_tools_test.go | 73 ++++++++ tools/bazel/go_repository_tools.bzl | 210 ++++++++++++++++++++++ 4 files changed, 300 insertions(+) create mode 100644 scripts/bazel_go_repository_tools_test.go create mode 100644 tools/bazel/go_repository_tools.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 5fb24db7e..68ded81bb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -188,6 +188,22 @@ go_sdk.nogo( nogo = "//tools/nogo", ) +# gazelle compiles fetch_repo and gazelle, which every go_deps repository +# runs, with `go build` in a repository rule before any go_deps fetch can +# start (18-24 s of every CI lane's loading). This substitute serves those +# reproducible binaries from the repository cache by pinned sha256 and builds +# them from source on a miss (tools/bazel/go_repository_tools.bzl). +gazelle_non_module_deps = use_extension("@gazelle//internal/bzlmod:non_module_deps.bzl", "non_module_deps") + +go_repository_tools = use_repo_rule("//tools/bazel:go_repository_tools.bzl", "go_repository_tools") + +go_repository_tools(name = "gazelle_go_repository_tools") + +override_repo( + gazelle_non_module_deps, + bazel_gazelle_go_repository_tools = "gazelle_go_repository_tools", +) + # gazelle.go.work joins the beads module and the nogo analyzers' module # (tools/nogo/go.mod) into one dependency graph for Bazel only; `go build` in # the repository root never reads it (it is not named go.work). diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index ef89f2709..36b26accd 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -37,6 +37,7 @@ go_test( "bazel_cache_mode_test.go", "bazel_farm_workflow_test.go", "bazel_fork_mode_test.go", + "bazel_go_repository_tools_test.go", "bazel_hermetic_cc_test.go", "bazel_policy_test.go", "bazel_rbe_prewarm_test.go", diff --git a/scripts/bazel_go_repository_tools_test.go b/scripts/bazel_go_repository_tools_test.go new file mode 100644 index 000000000..9e11a2d45 --- /dev/null +++ b/scripts/bazel_go_repository_tools_test.go @@ -0,0 +1,73 @@ +package scripts_test + +import ( + "regexp" + "strings" + "testing" +) + +// MODULE.bazel substitutes tools/bazel/go_repository_tools.bzl for +// gazelle's @bazel_gazelle_go_repository_tools, which serves fetch_repo and +// gazelle from the repository cache by pinned sha256 instead of compiling +// them in every fresh output base. A gazelle or Go SDK bump without new pins +// still builds (from source) but silently loses that, so the pins must name +// the versions MODULE.bazel pins. + +const goRepositoryToolsRule = "tools/bazel/go_repository_tools.bzl" + +var goRepositoryToolsNames = []string{"fetch_repo", "gazelle", "generate_repo_config"} + +func TestGoRepositoryToolsOverridesGazelle(t *testing.T) { + module := readPolicyFile(t, bazelPolicyRoot(t), hermeticCCModuleFile) + for _, want := range []string{ + `gazelle_non_module_deps = use_extension("@gazelle//internal/bzlmod:non_module_deps.bzl", "non_module_deps")`, + `go_repository_tools = use_repo_rule("//tools/bazel:go_repository_tools.bzl", "go_repository_tools")`, + `go_repository_tools(name = "gazelle_go_repository_tools")`, + } { + if !strings.Contains(module, want) { + t.Errorf("%s must contain %s", hermeticCCModuleFile, want) + } + } + override, err := moduleCall(module, "override_repo", "gazelle_non_module_deps") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(override, `bazel_gazelle_go_repository_tools = "gazelle_go_repository_tools"`) { + t.Errorf("override_repo(gazelle_non_module_deps, ...) must map bazel_gazelle_go_repository_tools to gazelle_go_repository_tools, got:\n%s", override) + } +} + +func TestGoRepositoryToolsPinnedForModuleVersions(t *testing.T) { + root := bazelPolicyRoot(t) + module := readPolicyFile(t, root, hermeticCCModuleFile) + gazelle := regexp.MustCompile(`(?m)^bazel_dep\(name = "gazelle", version = "([^"]+)"\)`).FindStringSubmatch(module) + if gazelle == nil { + t.Fatalf("%s: no bazel_dep(name = \"gazelle\", version = ...)", hermeticCCModuleFile) + } + sdk, err := moduleCall(module, "go_sdk.download", `name = "go_sdk"`) + if err != nil { + t.Fatal(err) + } + goVersion := regexp.MustCompile(`version = "([^"]+)"`).FindStringSubmatch(sdk) + if goVersion == nil { + t.Fatalf("%s: go_sdk.download(name = \"go_sdk\") has no version", hermeticCCModuleFile) + } + + // CI's lanes run on linux/amd64; that host must always be pinned. + key := gazelle[1] + "/go" + goVersion[1] + "/linux_amd64" + rule := readPolicyFile(t, root, goRepositoryToolsRule) + entry := regexp.MustCompile(`(?s)"` + regexp.QuoteMeta(key) + `": \{(.*?)\}`).FindStringSubmatch(rule) + if entry == nil { + t.Fatalf("%s: _PINS has no %q entry (gazelle %s, Go %s from %s); fetch @gazelle_go_repository_tools on linux/amd64 and copy the digests it prints", + goRepositoryToolsRule, key, gazelle[1], goVersion[1], hermeticCCModuleFile) + } + pins := quotedPairs(entry[1]) + if len(pins) != len(goRepositoryToolsNames) { + t.Errorf("_PINS[%q] = %v, want exactly %v", key, pins, goRepositoryToolsNames) + } + for _, tool := range goRepositoryToolsNames { + if !sha256HexRE.MatchString(pins[tool]) { + t.Errorf("_PINS[%q][%q] = %q, want a sha256", key, tool, pins[tool]) + } + } +} diff --git a/tools/bazel/go_repository_tools.bzl b/tools/bazel/go_repository_tools.bzl new file mode 100644 index 000000000..dd81b99b8 --- /dev/null +++ b/tools/bazel/go_repository_tools.bzl @@ -0,0 +1,210 @@ +"""gazelle's go_repository_tools, served from Bazel's repository cache. + +Every go_deps repository (go_repository) runs two host binaries that gazelle +compiles with `go build` in a repository rule of its own, +@bazel_gazelle_go_repository_tools: fetch_repo, which downloads the module, +and gazelle, which writes its BUILD files. Nothing else can start until that +build finishes, and a fresh output base (every CI lane) pays it in full: +13-24 s of client loading on a 2-4 vCPU runner, before any go_deps fetch. + +The build is reproducible (-trimpath, -buildvcs=false, CGO_ENABLED=0), so +for a given gazelle release, Go SDK and host its output is fixed bytes. +_PINS records those bytes' sha256. This rule, which MODULE.bazel substitutes +for gazelle's (override_repo), asks the repository cache for each pinned +digest first: a hit is copied out after Bazel re-hashes it, so a runner cache +restored from elsewhere can only ever yield the pinned bytes. On a miss (or +an unpinned gazelle, Go or host) it builds exactly as gazelle does, and when +the result matches the pins it enters the repository cache, so the next +fetch, and the next CI run that restores that cache, skips the build. + +A gazelle or Go SDK bump changes the key: until _PINS gets the new entry +(the rule prints the digests to add), fetches build from source, as before. +""" + +_TOOLS = ["fetch_repo", "gazelle", "generate_repo_config"] + +# "//_" -> {tool: sha256}. +_PINS = { + "0.53.0/go1.26.7/linux_amd64": { + "fetch_repo": "6780b48003af40859f849a4cd4ae291473565a057447591a6cbf111eb45ea9e1", + "gazelle": "4c1300d28f01dbed14661010e6beb8d3595425a4ac4c57b97fd094bcf9d21e9f", + "generate_repo_config": "6e21ac3f7af18822045718a70118a45ba9d48a964558c483ff013bbe569f6b2f", + }, + "0.53.0/go1.26.9/linux_amd64": { + "fetch_repo": "8b1288089311dc63c3d4b04c433d9ba11df5b09b5e8fdb1f8db0804f360ca6c0", + "gazelle": "1b764f6d9ad08e324bdcab429c3105815b8914f289d594acfc791bdac2bc2b2c", + "generate_repo_config": "8fab5073786aa587248822b73733a3edde6a33bd34d6ae69c274536811b5f74a", + }, +} + +# gazelle's go_repository_tools.bzl _GO_REPOSITORY_TOOLS_BUILD_FILE. +_BUILD_FILE = """\ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "fetch_repo", + srcs = ["bin/fetch_repo{ext}"], +) + +filegroup( + name = "gazelle", + srcs = ["bin/gazelle{ext}"], +) + +filegroup( + name = "generate_repo_config", + srcs = ["bin/generate_repo_config{ext}"], +) + +exports_files(["ROOT"]) +""" + +# Never fetched: a repository cache miss must fall through to the source +# build, not to the network. +_NO_URL = "file:///nonexistent/go_repository_tools" + +def _host(rctx): + os = rctx.os.name.lower() + if os.startswith("linux"): + os = "linux" + elif os.startswith("mac"): + os = "darwin" + elif os.startswith("windows"): + os = "windows" + arch = {"x86_64": "amd64", "aarch64": "arm64"}.get(rctx.os.arch, rctx.os.arch) + return os + "_" + arch + +def _gazelle_version(rctx): + for line in rctx.read(rctx.attr._gazelle_module).splitlines(): + line = line.strip() + if line.startswith("version = \""): + return line.split("\"")[1] + fail("no version in @gazelle//:MODULE.bazel") + +def _go(rctx, go, args, env, **kwargs): + """Runs go with exactly env, like gazelle's env_execute: no client GOFLAGS, GOARCH, ...""" + env = dict(env, GOROOT = str(go.dirname.dirname), GOTOOLCHAIN = "local") + if rctx.os.name.lower().startswith("windows"): + return rctx.execute([str(go)] + args, environment = env, **kwargs) + return rctx.execute(["env", "-i"] + ["%s=%s" % kv for kv in env.items()] + [str(go)] + args, **kwargs) + +def _build(rctx, go, gazelle_version, out_dir): + """gazelle's go_repository_tools build (its go.env, flags and packages), into out_dir.""" + scratch = rctx.path("_scratch") + env = { + "GOCACHE": str(scratch) + "/gocache", + "GOPATH": str(scratch) + "/gopath", + "GOROOT_FINAL": "GOROOT", + "CGO_ENABLED": "0", + "HOME": str(scratch), + } + for k in ["PATH", "GOPROXY", "TMP", "TMPDIR"]: + v = rctx.getenv(k) + if v: + env[k] = v + ldflags = [ + "-w", + "-s", + "-X", + "github.com/bazel-contrib/bazel-gazelle/v2/cmd/gazelle/update.BazelModuleVersion=" + gazelle_version, + "-X", + "github.com/bazel-contrib/bazel-gazelle/v2/cmd/gazelle/update.IsBazelModule=True", + ] + result = _go( + rctx, + go, + [ + "build", + "-o", + str(rctx.path(out_dir)) + "/", + "-ldflags", + " ".join(ldflags), + "-trimpath", + "-buildvcs=false", + "github.com/bazel-contrib/bazel-gazelle/v2/cmd/gazelle", + "github.com/bazelbuild/bazel-gazelle/cmd/fetch_repo", + "github.com/bazelbuild/bazel-gazelle/cmd/generate_repo_config", + ], + env, + working_directory = str(rctx.path(rctx.attr._gazelle_module).dirname), + ) + if result.return_code: + fail("building gazelle's go_repository tools: " + result.stderr) + + # The module cache is read-only; make it removable. + rctx.execute(["chmod", "-R", "u+w", str(scratch)]) + rctx.delete(scratch) + +def _sha256(rctx, path): + result = rctx.execute(["sha256sum", str(path)]) + return result.stdout.split(" ")[0] if result.return_code == 0 else "unknown" + +def _go_repository_tools_impl(rctx): + ext = ".exe" if rctx.os.name.lower().startswith("windows") else "" + go = rctx.path(Label("@go_sdk//:bin/go" + ext)) + rctx.watch(go) + result = _go(rctx, go, ["env", "GOVERSION"], {}) + if result.return_code: + fail("go env GOVERSION: " + result.stderr) + gazelle_version = _gazelle_version(rctx) + key = "%s/%s/%s" % (gazelle_version, result.stdout.strip(), _host(rctx)) + pins = _PINS.get(key) + names = {tool: "%s%s" % (tool, ext) for tool in _TOOLS} + + rctx.file("BUILD.bazel", _BUILD_FILE.format(ext = ext), executable = False) + rctx.file("ROOT", "", executable = False) + metadata = rctx.repo_metadata(reproducible = True) if hasattr(rctx, "repo_metadata") else None + + if not pins: + _build(rctx, go, gazelle_version, "bin") + sums = {tool: _sha256(rctx, rctx.path("bin/" + names[tool])) for tool in _TOOLS} + + # buildifier: disable=print + print("go_repository_tools: no pins for %s, built from source. To serve it from the repository cache, add %r: %r to _PINS in tools/bazel/go_repository_tools.bzl." % (key, key, sums)) + return metadata + + # The repository cache, by pinned digest. + if all([ + rctx.download( + url = _NO_URL, + output = "bin/" + names[tool], + sha256 = pins[tool], + executable = True, + allow_fail = True, + ).success + for tool in _TOOLS + ]): + return metadata + + # A miss: build, and let the downloader check each binary against its + # pin, which also stores a match in the repository cache. + rctx.delete("bin") + _build(rctx, go, gazelle_version, "_build") + stale = {} + for tool in _TOOLS: + built = rctx.path("_build/" + names[tool]) + if not rctx.download( + url = "file://" + str(built), + output = "bin/" + names[tool], + sha256 = pins[tool], + executable = True, + allow_fail = True, + ).success: + stale[tool] = _sha256(rctx, built) + if stale: + # buildifier: disable=print + print("go_repository_tools: %s built from source with digests %r, not _PINS[%r]; the build is used but not cached. Update _PINS in tools/bazel/go_repository_tools.bzl." % (key, stale, key)) + rctx.delete("bin") + for tool in _TOOLS: + rctx.symlink("_build/" + names[tool], "bin/" + names[tool]) + else: + rctx.delete("_build") + return metadata + +go_repository_tools = repository_rule( + implementation = _go_repository_tools_impl, + doc = "gazelle's go_repository_tools (fetch_repo, gazelle, generate_repo_config), served from the repository cache when pinned.", + attrs = { + "_gazelle_module": attr.label(default = "@gazelle//:MODULE.bazel"), + }, +) From b162e0a4821b1cbf16e6c3b360d0d994c0844a9f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 08:02:40 +0000 Subject: [PATCH 012/151] ci(bazel): cut embedded/test lane long shards (served template, bulk seed, rebalance) (#7416) * ci(bazel): cut embedded/test lane long shards (served template, bulk seed, rebalance) Embedded lane (bazel-embedded): - RunBatchGetterAcceptsExactlyTheCap seeds its 1000 issues through a new optional BatchGetterFixture.CreateIssues hook (one write) instead of 1000 single-row commits. Wired for the embedded, served and dolt-server legs; the UOW leg keeps the per-row fallback. Same rows, same GetMany assertions. TestBatchGetterContract 199s -> ~7s and TestServedBatchGetterContract 204s -> ~7s on rbe-west. - The served-surface harness (newServedEnv) copies one per-process, freshly initialized reference workspace instead of initializing an embedded Dolt schema per case, the same pristine-template pattern internal/storage/embeddeddolt's tests already use (bd-7q32v). //internal/httpclient:httpclient_served_test: 3,713 -> ~1,200 shard-seconds; 24 -> 16 shards; heaviest shard 390s -> ~108s. - //internal/storage/embeddeddolt:embeddeddolt_embedded_test: re-measured every test on rbe-west (scripts/ci/embedded_storage_test_durations.json) and repacked 15 -> 20 shards (packed heaviest ~110s; was 352s in CI). Test lane (bazel-test): - //internal/httpclient:httpclient_test: unsharded (137-228s) -> 2 shards (76s / 97s measured). Bead: ga-vnycm2.3 Co-Authored-By: Claude Opus 5.5 (1M context) * test(dolt): seed the BatchGetter cap case without prefix validation The dolt-server leg's shared test store is configured with prefix "test", so the batch CreateIssues path's prefix validation refused the fixture's bge-* ids (the per-row CreateIssue the other seeds use does not validate). Bead: ga-vnycm2.3 Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): shard //internal/storage/embeddeddolt:embeddeddolt_test 9 ways At 8 round-robin shards one shard drew the package's three heaviest tests (153s on rbe-west in this PR's test lane); 9 spreads them. Bead: ga-vnycm2.3 Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .../scripts/embedded-storage-test-shards.txt | 801 +++++++++--------- .github/workflows/bazel.yml | 37 +- backend/conformance/batch_getter_contract.go | 26 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 4 +- internal/httpclient/BUILD.bazel | 28 +- .../httpclient/served_batch_getter_test.go | 1 + internal/httpclient/served_harness_test.go | 47 +- internal/httpclient/served_test.go | 3 + .../dolt/batch_getter_contract_test.go | 9 + internal/storage/embeddeddolt/BUILD.bazel | 16 +- .../batch_getter_contract_test.go | 1 + .../ci/embedded_storage_test_durations.json | 754 +++++++++-------- scripts/shard_manifest_test.go | 6 +- 13 files changed, 918 insertions(+), 815 deletions(-) diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 96250f443..c92319f31 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -114,12 +114,12 @@ # # Format: # -# 15-shard split for the Bazel-only embedded-Dolt storage tier +# 20-shard split for the Bazel-only embedded-Dolt storage tier # (bazel-embedded in .github/workflows/bazel.yml), bin-packed # longest-processing-time-first by measured wall-time from # scripts/ci/embedded_storage_test_durations.json (see that file for provenance # and its "relative weight, not absolute SLA" caveat). -# Regenerate with scripts/ci/gen_embedded_storage_shard_manifest.py 15 +# Regenerate with scripts/ci/gen_embedded_storage_shard_manifest.py 20 # --weights=duration --write after adding or removing Test* functions in # internal/storage/embeddeddolt/*_test.go. --write is incremental: it # keeps every already-assigned test on its current shard and only places @@ -134,413 +134,418 @@ # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator or the durations file instead. -15 1 TestBatchApplyContract -15 1 TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow -15 1 TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting -15 1 TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn -15 1 TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure -15 1 TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID -15 1 TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix -15 1 TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState -15 1 TestEmbeddedMergeAndSettleMetadataConflict -15 1 TestEmbeddedMergeAndSettleRefusesUnknownTable -15 1 TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain -15 1 TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides -15 1 TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse -15 1 TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt -15 1 TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse -15 1 TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection -15 1 TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer -15 1 TestEmbeddedRoleFixtureKitHooksAreUsable -15 1 TestLifecycleCreateContract -15 1 TestMergeWithStrategy_OursKeepsLocal +20 1 TestCountsWithMissingWispPlaneTables +20 1 TestCreateBatchFastPathsMatchPerRow_Embedded +20 1 TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween +20 1 TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder +20 1 TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle +20 1 TestEmbeddedIssueOperationsUpdateLabelPatchValueRules +20 1 TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue +20 1 TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates +20 1 TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry +20 1 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor +20 1 TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue +20 1 TestIgnoredCursorHealResumeRespectsOperatorOverride +20 1 TestReadyCounterContract +20 1 TestReclaimRefusesForeignReplicaLease -15 2 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase -15 2 TestCleanGitRemoteCacheGarbage -15 2 TestCreateBatchFastPathsMatchPerRow_Embedded -15 2 TestDepRekeyMarkerStillInIgnoredSeries -15 2 TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles -15 2 TestDualWriteFixtureKitIsWired -15 2 TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded -15 2 TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges -15 2 TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop -15 2 TestEmbeddedDependencyEditorRefusesAGhostSource -15 2 TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane -15 2 TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe -15 2 TestEmbeddedIssueOperationsCreateAggregatesRelations -15 2 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers -15 2 TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites -15 2 TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge -15 2 TestEmbeddedRelationsDirectionSelectsTheInverseGraph -15 2 TestEmbeddedRelationsLeavesTheCallersRequestAlone -15 2 TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether -15 2 TestEmbeddedRemoteMigrateGate_BlocksReopen -15 2 TestGitRefPinsMatchDolt -15 2 TestHeartbeatRejectsWispEmbedded -15 2 TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore -15 2 TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern -15 2 TestLeaseLifecycleEmbedded -15 2 TestLenientGateWarningBody_ShapedDecisions -15 2 TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery -15 2 TestSchemaAfterInit -15 2 TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed -15 2 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword +20 2 TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed +20 2 TestEmbeddedConfigCommitStagesConfig +20 2 TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest +20 2 TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent +20 2 TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps +20 2 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering +20 2 TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes +20 2 TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder +20 2 TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne +20 2 TestEmbeddedRelationsRefusesTheZeroDirection +20 2 TestEmbeddedReopenIssueCategorySemantics +20 2 TestEmbeddedTransactionWispLifecyclePersistsDurableDependent +20 2 TestEventsJournalAccessor +20 2 TestGetDependentRecordsEmbedded +20 2 TestLifecycleCloseReopenContract +20 2 TestOpenReturnsCachedStore +20 2 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials -15 3 TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest -15 3 TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape -15 3 TestEmbeddedClaimerRefusesAWispIDAsNotFound -15 3 TestEmbeddedConfigCommitStagesConfig -15 3 TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes -15 3 TestEmbeddedDependencyEditorRefusalWritesNothing -15 3 TestEmbeddedDependencyEditorRemoveIsIdempotent -15 3 TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove -15 3 TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace -15 3 TestEmbeddedMergeAndSettleFKCascadeRepair -15 3 TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn -15 3 TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing -15 3 TestEmbeddedReaderReadyLimitBoundary -15 3 TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven -15 3 TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength -15 3 TestEmbeddedRenameKeepsDependencyIDsDeterministic -15 3 TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded -15 3 TestEventsJournalReadRefusesAnInteriorHole -15 3 TestForeignSkipDetailIsBounded -15 3 TestListWisps -15 3 TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict -15 3 TestPushCommitsPendingChangesFirst -15 3 TestUpdateMergeOps_DoltEngine_TwoIssues -15 3 TestWithPeerAuth_RestoresAbsentEnv +20 3 TestBatchApplyContract +20 3 TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget +20 3 TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate +20 3 TestEmbeddedDependencyEditorRefusalWritesNothing +20 3 TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp +20 3 TestEmbeddedIssueOperationsRequestValuesAreNotMutated +20 3 TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance +20 3 TestEmbeddedReaderDoesNotMutateTheCallerRequest +20 3 TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress +20 3 TestEmbeddedReadyClaimerSkipsIneligibleFrontRows +20 3 TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary +20 3 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff +20 3 TestGetIssueCommentsPageEmbedded +20 3 TestGetReadyWorkIncludeEphemeralAppliesWispFilters +20 3 TestJournalContract +20 3 TestSearchWithMissingWispPlaneTables -15 4 TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra -15 4 TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed -15 4 TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded -15 4 TestEmbeddedClaimIssueWrapperClaimsTheWispPlane -15 4 TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt -15 4 TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder -15 4 TestEmbeddedCommitToleratesNothingToCommit -15 4 TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest -15 4 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy -15 4 TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape -15 4 TestEmbeddedReaderListEmptyPageIsWellFormed -15 4 TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary -15 4 TestEmbeddedReopenIssueCategorySemantics -15 4 TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit -15 4 TestEventsAutoPruneThrottleSkipsTheSecondPass -15 4 TestEventsJournalRestartAndRetentionBoundary -15 4 TestIgnoredCursorHealNeverResurrectsRolledBackVersions -15 4 TestMain -15 4 TestMemoriesContract -15 4 TestPromoteClearsNoHistory -15 4 TestRemoteVerbPlainRemoteKeepsAmbientEnv -15 4 TestReopenIssue -15 4 TestWithPeerAuth -15 4 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials +20 4 TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes +20 4 TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded +20 4 TestEmbeddedDependencyEditorAttributesItsEventsToTheActor +20 4 TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses +20 4 TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose +20 4 TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge +20 4 TestEmbeddedMergeAndSettleReportsOperatorConflicts +20 4 TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing +20 4 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet +20 4 TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength +20 4 TestEventsSinceEmbedded +20 4 TestIgnoredCursorHealConvergesFromEveryCrashState +20 4 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle +20 4 TestReleaserContract +20 4 TestRemoteGitDataRefOnEngine +20 4 TestUpdateMergeOps_DoltEngine_TwoIssues +20 4 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword -15 5 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase -15 5 TestBlockingAnnotatorContract -15 5 TestBootstrapperContract -15 5 TestBuildDSN_WithDatabase -15 5 TestCrossRecordInvariantContract -15 5 TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch -15 5 TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween -15 5 TestEmbeddedDepRekeyConvergesRenameChain -15 5 TestEmbeddedDependencyEditorAcceptsADiamond -15 5 TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants -15 5 TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge -15 5 TestEmbeddedDependencyEditorValidationRefusalsWriteNothing -15 5 TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore -15 5 TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables -15 5 TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults -15 5 TestEmbeddedMergeAndSettleReportsOperatorConflicts -15 5 TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation -15 5 TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows -15 5 TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder -15 5 TestEmbeddedReaderListMaxRowsIsHonored -15 5 TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest -15 5 TestEmbeddedRelationsAnswersInThePinnedOrder -15 5 TestEventsAutoPruneBoundsTheJournalToTheFloors -15 5 TestExpectedRevisionContract -15 5 TestGetDependentRecordsForIssuesEmbedded -15 5 TestLargeBatchApplyWallClock_Embedded -15 5 TestLargeGraphCreateTiming_Embedded -15 5 TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance -15 5 TestOpenReturnsCachedStore -15 5 TestPartitionWispIDs -15 5 TestRetentionContract -15 5 TestWithPeerAuth_KeyMismatchFailsClosed +20 5 TestCommitAllOutOfBandChangeAndHonestNoOp +20 5 TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles +20 5 TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure +20 5 TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone +20 5 TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge +20 5 TestEmbeddedDependencyEditorValidationRefusalsWriteNothing +20 5 TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure +20 5 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender +20 5 TestEmbeddedMergeAndSettleMetadataConflict +20 5 TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch +20 5 TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt +20 5 TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary +20 5 TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants +20 5 TestEmbeddedRelationsResolvesAWispAnchor +20 5 TestIgnoredCursorHealNeverResurrectsRolledBackVersions +20 5 TestLifecycleCreateContract +20 5 TestPushCommitsPendingChangesFirst -15 6 TestCreateIssuesRejectStaleUpserts -15 6 TestDecryptPassword_KeyMismatchNamesTheLocalKey -15 6 TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit -15 6 TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim -15 6 TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges -15 6 TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate -15 6 TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes -15 6 TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion -15 6 TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim -15 6 TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay -15 6 TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState -15 6 TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin -15 6 TestEmbeddedSyncCommitsConfigBeforeMerge -15 6 TestGetIssue -15 6 TestHeartbeatMatchesActorAcrossSpellingEmbedded -15 6 TestHistory_NullTextColumns -15 6 TestIsBlockedBatchEmbedded -15 6 TestLargeBatchApplyStatementCounts356_Embedded -15 6 TestRemoteGitDataRefOnEngine -15 6 TestRenameLabel -15 6 TestRenameLabelMintsOneVersionPerTouchedBead -15 6 TestRunInTransactionPostCommitStageFailureIsIndeterminate -15 6 TestStatsReporterContract -15 6 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials -15 6 TestWithPeerAuth_RestoresEnvWhenCallbackFails +20 6 TestConcurrencyMultiProcess +20 6 TestConcurrentOpenReturnsCached +20 6 TestDualWriteFixtureKitIsWired +20 6 TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed +20 6 TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt +20 6 TestEmbeddedDependencyEditorAcceptsADiamond +20 6 TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape +20 6 TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone +20 6 TestEmbeddedLenientOpenWarning_DataBehind +20 6 TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder +20 6 TestEmbeddedReaderReadyDeferredAndEphemeralGates +20 6 TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer +20 6 TestEventsJournalRestartAndRetentionBoundary +20 6 TestGraphCounterContract +20 6 TestGraphWalkPlansHonorJoinHints +20 6 TestHistoryPresenceContract +20 6 TestPromoteClearsNoHistory +20 6 TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery -15 7 TestCommitAllOutOfBandChangeAndHonestNoOp -15 7 TestConcurrencyMultiProcess -15 7 TestConcurrentOpenReturnsCached -15 7 TestDualWriteContractFlagOff -15 7 TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty -15 7 TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy -15 7 TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge -15 7 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType -15 7 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane -15 7 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp -15 7 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender -15 7 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext -15 7 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet -15 7 TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType -15 7 TestEmbeddedReaderReadySortPoliciesOrderTheSameRows -15 7 TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists -15 7 TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne -15 7 TestEventsJournalAccessor -15 7 TestHistoryPresenceContract -15 7 TestJournalContract -15 7 TestLifecycleCloseReopenContract -15 7 TestReclaimAnyReplicaOverride -15 7 TestTreeWalkerContract -15 7 TestWithPeerAuth_DivergedRemoteURLFailsClosed +20 7 TestCreateIssuesRejectStaleUpserts +20 7 TestDecryptPassword_KeyMismatchNamesTheLocalKey +20 7 TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex +20 7 TestEmbeddedDependencyEditorSnapshotsTheRequest +20 7 TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn +20 7 TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps +20 7 TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged +20 7 TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn +20 7 TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary +20 7 TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage +20 7 TestEmbeddedRelationsDirectionSelectsTheInverseGraph +20 7 TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether +20 7 TestEmbeddedRelationsResolvesTheAnchorIDExactly +20 7 TestGetDependentRecordsForIssuesEmbedded +20 7 TestHeartbeatRejectsWispEmbedded +20 7 TestPromoteStillWorksWithCollisionGuard +20 7 TestUpdateMergeOps_DoltEngine +20 7 TestWithPeerAuth -15 8 TestAddLabel -15 8 TestBuildDSN_NoDatabase -15 8 TestCleanGitRemoteCacheGarbage_NoCacheDir -15 8 TestCreateBatchFastPathsMatchPerRowLarge_Embedded -15 8 TestDoltCommitResponseLossIsIndeterminate -15 8 TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed -15 8 TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed -15 8 TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex -15 8 TestEmbeddedClaimerRefusalMessagesCarryTheirFragments -15 8 TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb -15 8 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest -15 8 TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches -15 8 TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent -15 8 TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress -15 8 TestEmbeddedLenientOpenWarning_DataBehind -15 8 TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict -15 8 TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690 -15 8 TestEmbeddedMigration0058HealsAlreadyAffectedDatabase -15 8 TestEmbeddedOpenReadOnly_SkipsGateAndMigrations -15 8 TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset -15 8 TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters -15 8 TestEmbeddedReaderReadyDeferredAndEphemeralGates -15 8 TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded -15 8 TestEmbeddedRelationsResolvesTheAnchorIDExactly -15 8 TestEmbeddedSQLCommitResponseLossIsIndeterminate -15 8 TestGetReadyWorkIncludeEphemeralAppliesWispFilters -15 8 TestHasRepository -15 8 TestLenientGateWarning_ComposedOutput -15 8 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle -15 8 TestReclaimExpiredLeaseSurvivesRestartEmbedded -15 8 TestRemoteVerbPlainRemoteWithoutAuthPresentsNone -15 8 TestTransactionCleanupAfterSQLCommitIsIndeterminate +20 8 TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting +20 8 TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion +20 8 TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches +20 8 TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState +20 8 TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress +20 8 TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints +20 8 TestEmbeddedReaderListEmptyPageIsWellFormed +20 8 TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix +20 8 TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped +20 8 TestEmbeddedRelationsRefusesAnUnusableTypeFilter +20 8 TestEmbeddedRoleFixtureKitHooksAreUsable +20 8 TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit +20 8 TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges +20 8 TestGetNextChildID +20 8 TestListWisps +20 8 TestPreviewStoreRefusesLocalMetadataWrite +20 8 TestReclaimAnyReplicaOverride +20 8 TestStatsReporterContract +20 8 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials -15 9 TestBatchGetterContract -15 9 TestCommenterContract -15 9 TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes -15 9 TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting -15 9 TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState -15 9 TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes -15 9 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy -15 9 TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked -15 9 TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge -15 9 TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones -15 9 TestEmbeddedIssueOperationsCloseAndReopenAWisp -15 9 TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose -15 9 TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel -15 9 TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal -15 9 TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress -15 9 TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry -15 9 TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage -15 9 TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix -15 9 TestEventsSinceEmbedded -15 9 TestIgnoredCursorHealConvergesFromEveryCrashState -15 9 TestLegacyTrackedIgnoredCursorUnwedgesPull -15 9 TestMetadataCASContract -15 9 TestPromoteStillWorksWithCollisionGuard -15 9 TestReadyWorkWithMissingDeferredParentTables -15 9 TestUpdateMergeOps_DoltEngine +20 9 TestDeleterContract +20 9 TestDualWriteContractFlagOff +20 9 TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds +20 9 TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies +20 9 TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants +20 9 TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges +20 9 TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest +20 9 TestEmbeddedReaderListMaxRowsIsHonored +20 9 TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse +20 9 TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection +20 9 TestForeignSkipDetailIsBounded +20 9 TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore +20 9 TestLeaseLifecycleEmbedded +20 9 TestOpenAfterCloseCreatesNewStore +20 9 TestPartitionWispIDs +20 9 TestReadyWorkWithMissingDeferredParentTables +20 9 TestRemoteVerbsPresentStoredPeerCredentials +20 9 TestRunInTransactionPostCommitStageFailureIsIndeterminate +20 9 TestWithPeerAuth_MissingRemoteFailsClosed -15 10 TestBuildDSN_SpacesInPath -15 10 TestCleanGitRemoteCacheGarbage_Throttled -15 10 TestCreateIssues -15 10 TestCreateRejectsCrossTableIDCollision -15 10 TestDeleterContract -15 10 TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing -15 10 TestEmbeddedDepRekeyMergesFreshDuplicateEdges -15 10 TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder -15 10 TestEmbeddedDependencyEditorRefusesCrossPlaneCycle -15 10 TestEmbeddedDoltStoreIssueLifecycleBuildsOperations -15 10 TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables -15 10 TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp -15 10 TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies -15 10 TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure -15 10 TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes -15 10 TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566 -15 10 TestEmbeddedReaderDoesNotMutateTheCallerRequest -15 10 TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities -15 10 TestEmbeddedReadyClaimerSkipsIneligibleFrontRows -15 10 TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped -15 10 TestEmbeddedTempRebuildIntermediatesNeverReachHistory -15 10 TestEmbeddedTransactionWispLifecyclePersistsDurableDependent -15 10 TestEpochContract -15 10 TestGetReadyWorkMoleculeFilter -15 10 TestGraphWalkPlansHonorJoinHints -15 10 TestHelperProcess -15 10 TestIgnoredCursorHealResumeRespectsOperatorOverride -15 10 TestReclaimRefusesForeignReplicaLease -15 10 TestRemoteEntryPointsUseVersionControlOps -15 10 TestTransactionCleanupBeforeSQLCommitRemainsDefinite -15 10 TestVersionReconcilerContract -15 10 TestWithPeerAuth_MissingRemoteFailsClosed +20 10 TestEdgeReaderContract +20 10 TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape +20 10 TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState +20 10 TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes +20 10 TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes +20 10 TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace +20 10 TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel +20 10 TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField +20 10 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers +20 10 TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal +20 10 TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape +20 10 TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows +20 10 TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset +20 10 TestEmbeddedRelationsLeavesTheCallersRequestAlone +20 10 TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing +20 10 TestEventsJournalPruneNeverSplitsASkewedPair +20 10 TestGitRefPinsMatchDolt +20 10 TestIsBlockedBatchEmbedded +20 10 TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance +20 10 TestMain +20 10 TestRetentionContract -15 11 TestCycleDetectorContract -15 11 TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure -15 11 TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess -15 11 TestEmbeddedCheckedWrappersRouteTheirPreconditions -15 11 TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget -15 11 TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes -15 11 TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle -15 11 TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent -15 11 TestEmbeddedGetMergeBlockersOnAQuietStore -15 11 TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps -15 11 TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel -15 11 TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance -15 11 TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape -15 11 TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue -15 11 TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch -15 11 TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing -15 11 TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges -15 11 TestGetDependentRecordsEmbedded -15 11 TestGetLabels -15 11 TestGetNextChildID -15 11 TestGraphCounterContract -15 11 TestLargeBatchApplyStatementCountsClassic40_Embedded -15 11 TestPristineEmbeddedDoltFixtureRelocatesPrefixes -15 11 TestReadyWorkPageTotal -15 11 TestUpdateIssueType +20 11 TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory +20 11 TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy +20 11 TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex +20 11 TestEmbeddedClaimerRefusalMessagesCarryTheirFragments +20 11 TestEmbeddedClaimerRefusesAWispIDAsNotFound +20 11 TestEmbeddedDependencyEditorRemoveIsIdempotent +20 11 TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion +20 11 TestEmbeddedMigration0058HealsAlreadyAffectedDatabase +20 11 TestEmbeddedMigration0062FlipsEventsToIgnoredPlane +20 11 TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides +20 11 TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters +20 11 TestEmbeddedReaderReadyLimitBoundary +20 11 TestEmbeddedReadyClaimerEmptyFrontIsNormal +20 11 TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps +20 11 TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern +20 11 TestLifecycleUpdateContract +20 11 TestReadyWorkStatementBudget +20 11 TestRemoteVerbPlainRemoteKeepsAmbientEnv +20 11 TestRenameLabelMintsOneVersionPerTouchedBead -15 12 TestBatchCreatorContract -15 12 TestDualWriteContract -15 12 TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds -15 12 TestEmbeddedDependencyEditorAttributesItsEventsToTheActor -15 12 TestEmbeddedDependencyEditorSnapshotsTheRequest -15 12 TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape -15 12 TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction -15 12 TestEmbeddedIssueOperationsRequestValuesAreNotMutated -15 12 TestEmbeddedMergeAndSettleMemoryConfigConflict -15 12 TestEmbeddedReaderGetOptionalRowListsAreOffByDefault -15 12 TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes -15 12 TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary -15 12 TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants -15 12 TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet -15 12 TestEmbeddedRecomputeAllBlockedWiring -15 12 TestEmbeddedRelationsResolvesAWispAnchor -15 12 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges -15 12 TestLargeBatchApplyStatementCounts712_Embedded -15 12 TestLifecycleUpdateContract -15 12 TestPreviewStoreRefusesLocalMetadataWrite -15 12 TestQuerierContract -15 12 TestReadyWorkPageTotalFilterMatrix -15 12 TestRemoteVerbsPresentStoredPeerCredentials -15 12 TestSearchWithMissingWispPlaneTables -15 12 TestWorkspaceConfigContract +20 12 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase +20 12 TestBuildDSN_SpacesInPath +20 12 TestCleanGitRemoteCacheGarbage_NoCacheDir +20 12 TestCrossRecordInvariantContract +20 12 TestCycleDetectorContract +20 12 TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch +20 12 TestDualWriteContract +20 12 TestEmbeddedBlockedRecheckRecordsUnblockingWrites +20 12 TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded +20 12 TestEmbeddedClaimIssueWrapperClaimsTheWispPlane +20 12 TestEmbeddedCommitToleratesNothingToCommit +20 12 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType +20 12 TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove +20 12 TestEmbeddedDoltStoreIssueLifecycleBuildsOperations +20 12 TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape +20 12 TestEmbeddedIssueOperationsCloseAndReopenAWisp +20 12 TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge +20 12 TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists +20 12 TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities +20 12 TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit +20 12 TestEmbeddedSQLCommitResponseLossIsIndeterminate +20 12 TestEventsAutoPruneBoundsTheJournalToTheFloors +20 12 TestHasRepository +20 12 TestHistory_NullTextColumns +20 12 TestLenientGateWarningBody_ShapedDecisions +20 12 TestMetadataCASContract +20 12 TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed +20 12 TestWorkspaceConfigContract -15 13 TestCountsIncludeWispDependencies -15 13 TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch -15 13 TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp -15 13 TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing -15 13 TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone -15 13 TestEmbeddedDependencyEditorRefusesAMissingLocalTarget -15 13 TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged -15 13 TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses -15 13 TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates -15 13 TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary -15 13 TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField -15 13 TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566 -15 13 TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints -15 13 TestEmbeddedMigration0062FlipsEventsToIgnoredPlane -15 13 TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder -15 13 TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates -15 13 TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary -15 13 TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet -15 13 TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions -15 13 TestEmbeddedRelationsRefusesAnUnusableTypeFilter -15 13 TestEmbeddedRelationsRefusesTheZeroDirection -15 13 TestEventsJournal_EmbeddedConcurrentGaplessNoDup -15 13 TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree -15 13 TestIgnoredCursorHealRespectsOperatorOverride -15 13 TestLegacyTrackedIgnoredCursorHealsOnOpen -15 13 TestSweeperContract +20 13 TestAddDependency +20 13 TestAddLabel +20 13 TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing +20 13 TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder +20 13 TestEmbeddedDependencyEditorRefusesAMissingLocalTarget +20 13 TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge +20 13 TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent +20 13 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp +20 13 TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers +20 13 TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables +20 13 TestEmbeddedIssueOperationsCreateAggregatesRelations +20 13 TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing +20 13 TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded +20 13 TestEmbeddedRecomputeAllBlockedWiring +20 13 TestHeartbeatMatchesActorAcrossSpellingEmbedded +20 13 TestLegacyTrackedIgnoredCursorHealsOnOpen +20 13 TestMemoriesContract +20 13 TestQuerierContract +20 13 TestTreeWalkerContract -15 14 TestAddDependency -15 14 TestCounterContract -15 14 TestCountsWithMissingWispPlaneTables -15 14 TestEmbeddedBlockedRecheckFailureKeepsTheWrite -15 14 TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies -15 14 TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest -15 14 TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers -15 14 TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion -15 14 TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone -15 14 TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps -15 14 TestEmbeddedIssueOperationsUpdateLabelPatchValueRules -15 14 TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge -15 14 TestEmbeddedOpenForReadOnlyCommand_LenientGate -15 14 TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse -15 14 TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing -15 14 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits -15 14 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor -15 14 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff -15 14 TestMergeWithStrategy_TheirsResolvesAndCommits -15 14 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint -15 14 TestOpenAfterCloseCreatesNewStore -15 14 TestOpenDifferentDirsReturnsDifferentStores -15 14 TestPruneEventsBatchDeletesTheLowestSeqsOnly -15 14 TestReadyCounterContract -15 14 TestUnknownProvenanceLeaseStaysReclaimable +20 14 TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest +20 14 TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed +20 14 TestEmbeddedDepRekeyMergesFreshDuplicateEdges +20 14 TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes +20 14 TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes +20 14 TestEmbeddedDependencyEditorRefusesAGhostSource +20 14 TestEmbeddedDependencyEditorRefusesCrossPlaneCycle +20 14 TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked +20 14 TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants +20 14 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse +20 14 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane +20 14 TestEmbeddedGetMergeBlockersOnAQuietStore +20 14 TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones +20 14 TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict +20 14 TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse +20 14 TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree +20 14 TestGetLabels +20 14 TestPristineEmbeddedDoltFixtureRelocatesPrefixes +20 14 TestSchemaAfterInit +20 14 TestWithPeerAuth_RestoresAbsentEnv -15 15 TestCreateIssue -15 15 TestEdgeReaderContract -15 15 TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory -15 15 TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex -15 15 TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded -15 15 TestEmbeddedBlockedRecheckRecordsUnblockingWrites -15 15 TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne -15 15 TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants -15 15 TestEmbeddedDependencyEditorAcceptsAnExternalTarget -15 15 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped -15 15 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse -15 15 TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion -15 15 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering -15 15 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers -15 15 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 -15 15 TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke -15 15 TestEmbeddedOpenForRemoteSync_DataBehind -15 15 TestEmbeddedReadyClaimerEmptyFrontIsNormal -15 15 TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue -15 15 TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit -15 15 TestEventsJournalPruneNeverSplitsASkewedPair -15 15 TestEventsJournal_ActorMatrixOnEmbedded -15 15 TestGetIssueCommentsPageEmbedded -15 15 TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps -15 15 TestIgnoredCursorHealIsIdempotent -15 15 TestReadyWorkStatementBudget -15 15 TestReleaserContract +20 15 TestBuildDSN_NoDatabase +20 15 TestCleanGitRemoteCacheGarbage +20 15 TestCommenterContract +20 15 TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra +20 15 TestCreateBatchFastPathsMatchPerRowLarge_Embedded +20 15 TestDoltCommitResponseLossIsIndeterminate +20 15 TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing +20 15 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest +20 15 TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop +20 15 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy +20 15 TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane +20 15 TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies +20 15 TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel +20 15 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext +20 15 TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites +20 15 TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet +20 15 TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType +20 15 TestEmbeddedRenameKeepsDependencyIDsDeterministic +20 15 TestEventsJournalReadRefusesAnInteriorHole +20 15 TestEventsJournal_EmbeddedConcurrentGaplessNoDup +20 15 TestExpectedRevisionContract +20 15 TestLargeBatchApplyStatementCounts356_Embedded +20 15 TestLargeGraphCreateTiming_Embedded +20 15 TestReadyWorkPageTotalFilterMatrix +20 15 TestReclaimExpiredLeaseSurvivesRestartEmbedded +20 15 TestRemoteEntryPointsUseVersionControlOps +20 15 TestTransactionCleanupBeforeSQLCommitRemainsDefinite +20 15 TestUpdateIssueType + +20 16 TestCreateIssues +20 16 TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch +20 16 TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne +20 16 TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded +20 16 TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure +20 16 TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction +20 16 TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates +20 16 TestEmbeddedMergeAndSettleRefusesUnknownTable +20 16 TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation +20 16 TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes +20 16 TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions +20 16 TestEmbeddedTempRebuildIntermediatesNeverReachHistory +20 16 TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded +20 16 TestEventsJournal_ActorMatrixOnEmbedded +20 16 TestIgnoredCursorHealRespectsOperatorOverride +20 16 TestMergeWithStrategy_TheirsResolvesAndCommits +20 16 TestOpenDifferentDirsReturnsDifferentStores +20 16 TestRemoteVerbPlainRemoteWithoutAuthPresentsNone +20 16 TestUnknownProvenanceLeaseStaysReclaimable + +20 17 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase +20 17 TestBuildDSN_WithDatabase +20 17 TestCleanGitRemoteCacheGarbage_Throttled +20 17 TestDepRekeyMarkerStillInIgnoredSeries +20 17 TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty +20 17 TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess +20 17 TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit +20 17 TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting +20 17 TestEmbeddedDepRekeyConvergesRenameChain +20 17 TestEmbeddedDependencyEditorAcceptsAnExternalTarget +20 17 TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb +20 17 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy +20 17 TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore +20 17 TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables +20 17 TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain +20 17 TestEmbeddedReaderGetOptionalRowListsAreOffByDefault +20 17 TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse +20 17 TestEmbeddedReaderReadySortPoliciesOrderTheSameRows +20 17 TestEmbeddedRelationsAnswersInThePinnedOrder +20 17 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges +20 17 TestEpochContract +20 17 TestHelperProcess +20 17 TestLegacyTrackedIgnoredCursorUnwedgesPull +20 17 TestLenientGateWarning_ComposedOutput +20 17 TestReadyWorkPageTotal +20 17 TestSweeperContract +20 17 TestTransactionCleanupAfterSQLCommitIsIndeterminate +20 17 TestWithPeerAuth_RestoresEnvWhenCallbackFails + +20 18 TestBatchCreatorContract +20 18 TestBlockingAnnotatorContract +20 18 TestCountsIncludeWispDependencies +20 18 TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp +20 18 TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges +20 18 TestEmbeddedBlockedRecheckFailureKeepsTheWrite +20 18 TestEmbeddedCheckedWrappersRouteTheirPreconditions +20 18 TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe +20 18 TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID +20 18 TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim +20 18 TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary +20 18 TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690 +20 18 TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566 +20 18 TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState +20 18 TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin +20 18 TestLargeBatchApplyWallClock_Embedded +20 18 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint +20 18 TestRenameLabel +20 18 TestVersionReconcilerContract + +20 19 TestBatchGetterContract +20 19 TestBootstrapperContract +20 19 TestCreateIssue +20 19 TestCreateRejectsCrossTableIDCollision +20 19 TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim +20 19 TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge +20 19 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers +20 19 TestEmbeddedMergeAndSettleMemoryConfigConflict +20 19 TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566 +20 19 TestEmbeddedOpenForReadOnlyCommand_LenientGate +20 19 TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest +20 19 TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven +20 19 TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet +20 19 TestEmbeddedSyncCommitsConfigBeforeMerge +20 19 TestGetIssue +20 19 TestGetReadyWorkMoleculeFilter +20 19 TestLargeBatchApplyStatementCounts712_Embedded +20 19 TestReopenIssue +20 19 TestWithPeerAuth_DivergedRemoteURLFailsClosed +20 19 TestWithPeerAuth_KeyMismatchFailsClosed + +20 20 TestCounterContract +20 20 TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow +20 20 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped +20 20 TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion +20 20 TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults +20 20 TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix +20 20 TestEmbeddedMergeAndSettleFKCascadeRepair +20 20 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 +20 20 TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke +20 20 TestEmbeddedOpenForRemoteSync_DataBehind +20 20 TestEmbeddedOpenReadOnly_SkipsGateAndMigrations +20 20 TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay +20 20 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits +20 20 TestEmbeddedRemoteMigrateGate_BlocksReopen +20 20 TestEventsAutoPruneThrottleSkipsTheSecondPass +20 20 TestIgnoredCursorHealIsIdempotent +20 20 TestLargeBatchApplyStatementCountsClassic40_Embedded +20 20 TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict +20 20 TestMergeWithStrategy_OursKeepsLocal +20 20 TestPruneEventsBatchDeletesTheLowestSeqsOnly diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5ca224450..eb761aa7f 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1283,8 +1283,8 @@ jobs: name: embedded Dolt tier # Remote execution only: skipped unless the rbe job chose remote, # fork-ro or fork-rw (enabled; not in modes cache, local or skip). - # Locally the tier is 67 race test - # processes (50 cmd/bd shards + 15 embeddeddolt storage shards + 2 + # Locally the tier is 72 race test + # processes (50 cmd/bd shards + 20 embeddeddolt storage shards + 2 # conformance partitions; ~24k test-seconds measured locally -- see # scripts/ci/embedded_cmd_test_durations.json and # scripts/ci/embedded_storage_test_durations.json -- up from an earlier, @@ -1293,26 +1293,21 @@ jobs: # proves nothing pr-risk.yml's embedded jobs (which do run for forks) do # not. # - # Shard counts: storage (15) is sized the way both tiers were originally - # meant to be -- the smallest count whose LPT-packed heaviest shard - # (scripts/ci/gen_embedded_storage_shard_manifest.py --weights=duration - # --repack) lands at or below the tier's own heaviest single top-level - # test (211-225s). cmd cannot reach that floor: Bazel hard-caps - # shard_count at 50 per sh_test-like rule ("Having more than 50 shards is - # indicative of poor test organization. Please reduce the number of - # shards." -- a native analysis-time error, not a style lint), and at 50 - # shards the cmd suite's LPT-packed heaviest shard is ~409.6s -- above - # its own single heaviest test's 239.7s, because the suite's total - # duration (~20145 test-seconds across 242 tests) is too large to flatten - # to the single-test floor within 50 bins. Clearing more shards would - # need either further per-test speedups spread across the suite or + # Shard counts: storage (20) is the smallest count whose LPT-packed + # heaviest shard (scripts/ci/gen_embedded_storage_shard_manifest.py + # --weights=duration --repack, on rbe-west measurements) stays near 110s, + # under the ~150s per-shard budget with headroom for executor load; its + # heaviest single top-level test is ~90s. cmd (50) sits at Bazel's hard + # cap per sh_test-like rule ("Having more than 50 shards is indicative of + # poor test organization. Please reduce the number of shards." -- a + # native analysis-time error, not a style lint); on rbe-west its shards + # ran 83-209s (6.5k shard-seconds, 2026-10-09), so it, not storage or + # //internal/httpclient:httpclient_served_test, is this lane's floor. + # Lowering it needs per-test speedups spread across the suite or # splitting bd_embedded_test into multiple <=50-shard sh_test rules with # shard-index-offset plumbing through tools/bazel/go_test_manifest_shard.sh - # and check_shard_coverage.py; neither is done here, so this lane's floor - # is cmd's ~409.6s, not the ~240s a 5-minute lane (including this job's - # ~60s fixed cost) would need -- tracked as a follow-up, not silently - # hidden in the shard count. On RBE this makes the tier's wall time ~= - # that floor + fixed cost (checkout, build, shard dispatch), not ~= + # and check_shard_coverage.py. On RBE the tier's wall time ~= that floor + # + fixed cost (checkout, build, shard dispatch), not ~= # sum(durations)/shard_count the way a CPU-core-bound local sweep is -- # each shard is its own remote action and runs concurrently with every # other shard, farm capacity permitting. @@ -1399,7 +1394,7 @@ jobs: python3 tools/bazel/check_shard_coverage.py --bep "$RUNNER_TEMP/bazel-bep.json" --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 50 - --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 15 + --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 20 --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test diff --git a/backend/conformance/batch_getter_contract.go b/backend/conformance/batch_getter_contract.go index 6cfdd6590..f64dabde6 100644 --- a/backend/conformance/batch_getter_contract.go +++ b/backend/conformance/batch_getter_contract.go @@ -36,6 +36,13 @@ type BatchGetterFixture struct { BatchGetter publicops.BatchGetter // CreateIssue seeds a durable issue in the issues plane. CreateIssue func(context.Context, *types.Issue, string) error + // CreateIssues seeds many durable issues in one call. Optional: only + // RunBatchGetterAcceptsExactlyTheCap uses it, to seed its MaxGetManyIDs + // issues in one write instead of MaxGetManyIDs single-row commits (about + // 200s of the embedded suites' race runs); nil falls back to CreateIssue + // per row. The seeded rows are the same either way, and the case asserts + // only on the GetMany read. + CreateIssues func(context.Context, []*types.Issue, string) error // CreateWisp seeds an ephemeral issue in the wisps plane. It is a separate // field rather than an Ephemeral flag on CreateIssue because the three // adapters reach the two planes through different verbs. @@ -205,7 +212,7 @@ func RunBatchGetterAcceptsExactlyTheCap(t *testing.T, ctx context.Context, fixtu for i := range ids { ids[i] = fmt.Sprintf("%s-atcap-%d", fixture.IssuePrefix, i) } - seedBatchGetterIssues(t, ctx, fixture, ids...) + seedBatchGetterIssuesInBulk(t, ctx, fixture, ids...) result := getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: ids}) if len(result.Issues) != publicops.MaxGetManyIDs { @@ -363,6 +370,23 @@ func seedBatchGetterIssues(t *testing.T, ctx context.Context, fixture BatchGette } } +// seedBatchGetterIssuesInBulk seeds the same rows as seedBatchGetterIssues, +// through the fixture's CreateIssues hook when it has one. +func seedBatchGetterIssuesInBulk(t *testing.T, ctx context.Context, fixture BatchGetterFixture, ids ...string) { + t.Helper() + if fixture.CreateIssues == nil { + seedBatchGetterIssues(t, ctx, fixture, ids...) + return + } + issues := make([]*types.Issue, len(ids)) + for i, id := range ids { + issues[i] = batchGetterSeed(id, false) + } + if err := fixture.CreateIssues(ctx, issues, "batch-getter-seed"); err != nil { + t.Fatalf("seed %d issues: %v", len(issues), err) + } +} + func seedBatchGetterWisp(t *testing.T, ctx context.Context, fixture BatchGetterFixture, id string) { t.Helper() if err := fixture.CreateWisp(ctx, batchGetterSeed(id, true), "batch-getter-seed"); err != nil { diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index a9f34e510..15ba5624c 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1039,7 +1039,7 @@ Required` requires them to have run remotely and passed. - `tools/bazel/check_shard_coverage.py` runs after each tier. It requires: - every Bazel shard of `//cmd/bd:bd_embedded_test` (50; the manifest's frozen 20-shard block was the retired `test-embedded-cmd` jobs' split, - F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (15; + F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; the frozen 5-shard block was `test-embedded-storage`'s, F1), `//cmd/bd:bd_proxied_test` (30; the frozen 15-shard block was `test-proxied-cmd`'s, F2) and @@ -1071,7 +1071,7 @@ Required` requires them to have run remotely and passed. Its Tier 3, the served HTTP corpus (`./internal/httpclient` with `BEADS_TEST_EMBEDDED_DOLT=1` and `BEADS_HTTP_TEST_REQUIRED=1`), runs in the embedded lane as `//internal/httpclient:httpclient_served_test` - (race build, 24 shards). + (race build, 16 shards). `docs-mintlify.yml` likewise drops its docsync job (`go test ./test/docsync`, which `bazel-test` runs as `//test/docsync:docsync_test`) and keeps only Mintlify's network-bound broken-link check. diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index beffe7048..8e8bc8be3 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -79,11 +79,12 @@ go_library( go_test( name = "httpclient_test", - # Unsharded, this package takes ~350s under --config=ci (BEADS_TEST_SKIP=dolt, no - # BEADS_TEST_EMBEDDED_DOLT): mostly wall-clock wait on the CA tests' dialer/keep-alive - # timeouts, not CPU. The go_test default size (medium, 300s timeout) leaves no margin - # and times out intermittently; size=large/timeout=long follows the same convention as - # //internal/storage/embeddeddolt's go_test. + # Unsharded, this package took 137-228s on rbe-west under --config=ci + # (BEADS_TEST_SKIP=dolt, no BEADS_TEST_EMBEDDED_DOLT; 2026-10-09), the test lane's + # longest single action. Two shards (rules_go deals tests out round-robin) put + # ~68s of measured test time on each; the three heaviest tests (31s, 20s, 13s) + # land on different shards. size=large/timeout=long follows the same convention + # as //internal/storage/embeddeddolt's go_test. size = "large", timeout = "long", srcs = [ @@ -171,6 +172,7 @@ go_test( # `make bazel-sync` (the directive just says so). data = glob(["*.go"]), embed = [":httpclient"], + shard_count = 2, deps = [ "//backend/conformance", "//internal/eventsjournal", @@ -203,12 +205,14 @@ go_test( # fails each served case rather than skipping it. It is the script's run on # the lane's race build (raceEnabled shrinks its one large apply). # -# Shard count: on the lane's executors (2026-10-08) the package is 702 tests -# and ~3,700 test-seconds, the heaviest, TestServedBatchGetterContract, at -# 170-230s. rules_go deals the tests out round-robin, so the shard that draws -# it runs it plus about a 24th of the rest: 283s measured, ~370s at the slow -# end. Both are under the ~410s cmd/bd shard that sets the lane's floor (see -# bazel.yml's bazel-embedded), so this target does not lengthen the lane. +# Shard count: on the lane's executors (rbe-west, 2026-10-09) the package is +# 725 tests and ~1,300 test-seconds, the heaviest about 54s. Each served case +# copies one per-process template workspace instead of initializing its own +# embedded Dolt schema (served_harness_test.go's servedTemplateWorkspace), +# and TestServedBatchGetterContract seeds its 1000 issues in one write: before +# both, the package was ~3,700 test-seconds over 24 shards, the heaviest shard +# 390s. rules_go deals the tests out round-robin; at 16 shards the measured +# loads come to 62-112s per shard. # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, @@ -233,7 +237,7 @@ sh_test( "BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1", }, - shard_count = 24, + shard_count = 16, tags = ["embedded"], ) diff --git a/internal/httpclient/served_batch_getter_test.go b/internal/httpclient/served_batch_getter_test.go index 746167eb0..18813fc01 100644 --- a/internal/httpclient/served_batch_getter_test.go +++ b/internal/httpclient/served_batch_getter_test.go @@ -35,6 +35,7 @@ func TestServedBatchGetterContract(t *testing.T) { IssuePrefix: "bge", BatchGetter: getter, CreateIssue: e.createIssue, + CreateIssues: e.reference.CreateIssues, CreateWisp: e.createWisp, CountHistory: e.countHistory, } diff --git a/internal/httpclient/served_harness_test.go b/internal/httpclient/served_harness_test.go index 3b11d9749..917335a2a 100644 --- a/internal/httpclient/served_harness_test.go +++ b/internal/httpclient/served_harness_test.go @@ -7,10 +7,12 @@ package httpclient import ( "context" + "fmt" "io" "net/url" "os" "path/filepath" + "sync" "testing" "time" @@ -44,7 +46,10 @@ import ( // per case costs about a second; sharing one across cases would make an id // collision between two contracts a debugging session instead of a rename, and // the contracts namespace their ids by prefix precisely so they can be run this -// way. +// way. Each case's engine is a private copy of one freshly initialized +// workspace (servedTemplateWorkspace), not a fresh init of its own: creating +// the schema is most of a case's cost under -race, and it is the same schema +// every time. // servedEnv is one client-server-store triple and the hooks a fixture needs. type servedEnv struct { @@ -120,7 +125,14 @@ func newServedEnvExpecting(t *testing.T, prefix, expectID string, opts ...func(* t.Helper() skipUnlessEmbeddedDolt(t) + template, err := servedTemplateWorkspace() + if err != nil { + t.Fatalf("initialize the reference workspace template: %v", err) + } beadsDir := t.TempDir() + if err := os.CopyFS(beadsDir, os.DirFS(template)); err != nil { + t.Fatalf("copy the reference workspace template: %v", err) + } ctx := t.Context() reference, err := embeddeddolt.Open(ctx, beadsDir, servedDatabase, "main") if err != nil { @@ -176,6 +188,39 @@ func newServedEnvExpecting(t *testing.T, prefix, expectID string, opts ...func(* } } +var ( + servedTemplateOnce sync.Once + servedTemplateDir string + servedTemplateErr error +) + +// servedTemplateWorkspace initializes one reference workspace per test process +// (embeddeddolt.Open on an empty directory: the schema and every migration) and +// closes it, so each newServedEnv can copy the closed workspace and reopen it +// instead of initializing its own. TestMain removes it after the last case. +func servedTemplateWorkspace() (string, error) { + servedTemplateOnce.Do(func() { + dir, err := os.MkdirTemp("", "httpclient-served-template-") + if err != nil { + servedTemplateErr = err + return + } + store, err := embeddeddolt.Open(context.Background(), dir, servedDatabase, "main") + if err != nil { + _ = os.RemoveAll(dir) + servedTemplateErr = fmt.Errorf("open %s: %w", dir, err) + return + } + if err := store.Close(); err != nil { + _ = os.RemoveAll(dir) + servedTemplateErr = fmt.Errorf("close %s: %w", dir, err) + return + } + servedTemplateDir = dir + }) + return servedTemplateDir, servedTemplateErr +} + // serveConfig binds every served role to the reference store. // // Every one of them is taken through the store's own capability accessor rather diff --git a/internal/httpclient/served_test.go b/internal/httpclient/served_test.go index 5bc182b82..5037cb1f7 100644 --- a/internal/httpclient/served_test.go +++ b/internal/httpclient/served_test.go @@ -73,6 +73,9 @@ func TestMain(m *testing.M) { if compositionStop != nil { compositionStop() } + if servedTemplateDir != "" { + _ = os.RemoveAll(servedTemplateDir) + } os.Exit(code) } diff --git a/internal/storage/dolt/batch_getter_contract_test.go b/internal/storage/dolt/batch_getter_contract_test.go index 9fc0ff654..9fe6fb8f6 100644 --- a/internal/storage/dolt/batch_getter_contract_test.go +++ b/internal/storage/dolt/batch_getter_contract_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" ) // TestBatchGetterContract runs the BatchGetter contract against the @@ -77,10 +79,17 @@ func newDoltBatchGetterFixture(t *testing.T, prefix string) (conformance.BatchGe t.Fatalf("BatchGetter(): %v", err) } kit := newDoltRoleFixtureKit(store, prefix) + // The shared test store's configured prefix is not the fixture's, and the + // batch path validates prefixes where the per-row CreateIssue every other + // seed uses does not; skip it so both paths seed the same rows. + createIssues := func(ctx context.Context, issues []*types.Issue, actor string) error { + return store.CreateIssuesWithFullOptions(ctx, issues, actor, storage.BatchCreateOptions{SkipPrefixValidation: true}) + } fixture := conformance.BatchGetterFixture{ IssuePrefix: kit.IssuePrefix, BatchGetter: getter, CreateIssue: kit.CreateIssue, + CreateIssues: createIssues, CreateWisp: kit.CreateWisp, CountHistory: kit.CountHistory, } diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index e4e9258db..64e516d60 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -94,7 +94,11 @@ go_library( go_test( name = "embeddeddolt_test", # -race -short runs ~6 min unsharded (pr-core gives go test -timeout=30m). - # Bazel shards by test count, not duration. + # Bazel shards by test count, not duration: rules_go deals tests out + # round-robin. At 8 shards one shard drew TestWithPeerAuth, + # TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword and + # TestPushCommitsPendingChangesFirst (100-153s on rbe-west); 9 spreads + # them (measured costs put the heaviest shard at ~87s). size = "large", timeout = "long", srcs = [ @@ -230,7 +234,7 @@ go_test( "workspace_config_contract_test.go", ], embed = [":embeddeddolt"], - shard_count = 8, + shard_count = 9, deps = [ "//:beads", "//backend/conformance", @@ -265,10 +269,10 @@ go_test( ], ) -# Runs 15 Bazel shards against every top-level Test* function in +# Runs 20 Bazel shards against every top-level Test* function in # internal/storage/embeddeddolt/*_test.go but TestConformance, assigned by -# the manifest's 15-shard block (duration-balanced; see -# scripts/ci/embedded_storage_test_durations.json), with +# the manifest's 20-shard block (duration-balanced on rbe-west measurements; +# see scripts/ci/embedded_storage_test_durations.json), with # BEADS_TEST_EMBEDDED_DOLT=1; the wrapper points # BEADS_TEST_EMBEDDED_TEST_BINARY at this binary, which the concurrency test # re-executes. This is a *different* shard count from PR Risk's legacy @@ -304,7 +308,7 @@ sh_test( "//:.github/scripts/embedded-storage-test-shards.txt", ], env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, - shard_count = 15, + shard_count = 20, tags = ["embedded"], ) diff --git a/internal/storage/embeddeddolt/batch_getter_contract_test.go b/internal/storage/embeddeddolt/batch_getter_contract_test.go index 2da37613d..3bf28b2b4 100644 --- a/internal/storage/embeddeddolt/batch_getter_contract_test.go +++ b/internal/storage/embeddeddolt/batch_getter_contract_test.go @@ -76,6 +76,7 @@ func newEmbeddedBatchGetterFixture(t *testing.T, te *testEnv, prefix string) con IssuePrefix: kit.IssuePrefix, BatchGetter: getter, CreateIssue: kit.CreateIssue, + CreateIssues: te.store.CreateIssues, CreateWisp: kit.CreateWisp, CountHistory: kit.CountHistory, } diff --git a/scripts/ci/embedded_storage_test_durations.json b/scripts/ci/embedded_storage_test_durations.json index 92c4c4ede..7e3ae6350 100644 --- a/scripts/ci/embedded_storage_test_durations.json +++ b/scripts/ci/embedded_storage_test_durations.json @@ -1,17 +1,21 @@ { "_comment": [ - "Measured locally with `bazel --nohome_rc test --config=embedded", - "//internal/storage/embeddeddolt:embeddeddolt_embedded_test", - "--local_test_jobs=8` alongside the cmd-tier and conformance targets (all", - "four run in the same invocation, the same way bazel.yml's bazel-embedded", - "tier runs its shards concurrently on one host) -- race,", - "`-test.parallel=4`, tag filter `embedded`, run with the real committed", - "14-shard manifest (.github/scripts/embedded-storage-test-shards.txt) so", - "every one of the 387 top-level Test* functions (excluding TestConformance,", - "sharded separately, and TestMain, which is not dispatched through", - "-test.run and carries no measurable duration of its own) that existed at", - "capture time ran exactly once (local sandboxed execution only; no remote", - "executor or cache flags were configured for this measurement run).", + "Measured on rbe-west, the lane's own remote executors (instance oss), on", + "2026-10-09: two `bazel test --config=embedded --nocache_test_results", + "//internal/storage/embeddeddolt:embeddeddolt_embedded_test` runs (race,", + "`-test.parallel=4`), one with the then-committed 15-shard manifest and one", + "with an 18-shard repack, so every one of the 395 top-level Test* functions", + "(excluding TestConformance, sharded separately, and TestMain) ran exactly", + "once per run. Each value is the mean of the two runs; the second ran under", + "heavier executor load and came out ~25% slower across the board, which is", + "the spread to expect between CI runs. Both runs were taken after", + "RunBatchGetterAcceptsExactlyTheCap started seeding its 1000 issues in one", + "CreateIssues call (TestBatchGetterContract: 198.87s -> ~7s on the same", + "executors). The previous capture was local sandboxed execution, and it", + "predated TestBatchGetterContract, so the packer weighted that test at its", + "t.Run fallback (~54s) and stacked a ~200s test onto an already full shard", + "(shard 9 of 15: 352s in CI). Remote shard wall time tracked the packed", + "cost to within 2% in both runs (correlation 0.999+).", "", "Each value is that top-level test's own from Bazel's", "generated test.xml, NOT the enclosing , for the same", @@ -76,394 +80,402 @@ "changes all mean a shard's total packed cost is a planning estimate, not", "a guarantee." ], - "seconds_per_init_fallback": 3.8877, + "seconds_per_init_fallback": 2.416, "tests": { "TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase": 0.0, "TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase": 0.0, - "TestAddDependency": 33.47, - "TestAddLabel": 1.12, - "TestBatchApplyContract": 151.73, - "TestBatchCreatorContract": 11.81, - "TestBlockingAnnotatorContract": 17.98, - "TestBootstrapperContract": 3.79, + "TestAddDependency": 24.83, + "TestAddLabel": 0.61, + "TestBatchApplyContract": 62.95, + "TestBatchCreatorContract": 17.7, + "TestBatchGetterContract": 6.37, + "TestBlockingAnnotatorContract": 10.55, + "TestBootstrapperContract": 2.03, "TestBuildDSN_NoDatabase": 0.0, "TestBuildDSN_SpacesInPath": 0.0, "TestBuildDSN_WithDatabase": 0.0, "TestCleanGitRemoteCacheGarbage": 0.0, "TestCleanGitRemoteCacheGarbage_NoCacheDir": 0.0, "TestCleanGitRemoteCacheGarbage_Throttled": 0.0, - "TestCommenterContract": 25.32, - "TestCommitAllOutOfBandChangeAndHonestNoOp": 1.1, - "TestConcurrencyMultiProcess": 6.48, - "TestConcurrentOpenReturnsCached": 18.03, - "TestCounterContract": 10.53, - "TestCountsIncludeWispDependencies": 1.73, - "TestCountsWithMissingWispPlaneTables": 3.29, - "TestCreateIssue": 35.89, - "TestCreateIssues": 19.14, - "TestCreateIssuesRejectStaleUpserts": 9.47, - "TestCreateRejectsCrossTableIDCollision": 2.63, + "TestCommenterContract": 4.51, + "TestCommitAllOutOfBandChangeAndHonestNoOp": 0.58, + "TestConcurrencyMultiProcess": 4.32, + "TestConcurrentOpenReturnsCached": 11.14, + "TestCounterContract": 21.42, + "TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra": 0.8, + "TestCountsIncludeWispDependencies": 1.12, + "TestCountsWithMissingWispPlaneTables": 1.93, + "TestCreateBatchFastPathsMatchPerRowLarge_Embedded": 0.0, + "TestCreateBatchFastPathsMatchPerRow_Embedded": 89.57, + "TestCreateIssue": 21.51, + "TestCreateIssues": 16.54, + "TestCreateIssuesRejectStaleUpserts": 3.85, + "TestCreateRejectsCrossTableIDCollision": 1.54, "TestCrossRecordInvariantContract": 0.0, - "TestCycleDetectorContract": 14.23, - "TestDecryptPassword_KeyMismatchNamesTheLocalKey": 18.0, - "TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch": 17.16, - "TestDeleterContract": 61.52, + "TestCycleDetectorContract": 7.24, + "TestDecryptPassword_KeyMismatchNamesTheLocalKey": 10.81, + "TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch": 11.12, + "TestDeleterContract": 42.31, "TestDepRekeyMarkerStillInIgnoredSeries": 0.0, - "TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles": 4.78, + "TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles": 1.83, "TestDoltCommitResponseLossIsIndeterminate": 0.0, - "TestDualWriteContract": 2.41, - "TestDualWriteContractFlagOff": 0.74, - "TestDualWriteFixtureKitIsWired": 1.06, - "TestEdgeReaderContract": 10.02, - "TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing": 1.9, - "TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory": 19.12, - "TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes": 20.46, - "TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure": 19.63, - "TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed": 2.3, - "TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed": 0.79, - "TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty": 1.36, - "TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch": 22.31, - "TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest": 19.41, - "TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex": 1.64, - "TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp": 20.52, - "TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy": 4.15, - "TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess": 3.81, - "TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape": 21.59, - "TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex": 22.63, - "TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit": 22.49, - "TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded": 1.99, - "TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing": 18.68, - "TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed": 9.26, - "TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim": 1.89, - "TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges": 1.79, - "TestEmbeddedBlockedRecheckFailureKeepsTheWrite": 20.58, - "TestEmbeddedBlockedRecheckRecordsUnblockingWrites": 21.27, - "TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded": 22.27, - "TestEmbeddedCheckedWrappersRouteTheirPreconditions": 22.28, - "TestEmbeddedClaimIssueWrapperClaimsTheWispPlane": 1.35, - "TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne": 1.81, - "TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow": 1.13, - "TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone": 2.21, - "TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting": 1.6, - "TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting": 1.43, - "TestEmbeddedClaimerRefusalMessagesCarryTheirFragments": 1.7, - "TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt": 1.43, - "TestEmbeddedClaimerRefusesAWispIDAsNotFound": 1.02, - "TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState": 1.07, - "TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween": 1.81, - "TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder": 2.5, - "TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds": 18.89, - "TestEmbeddedCommitToleratesNothingToCommit": 1.12, - "TestEmbeddedConfigCommitStagesConfig": 0.55, - "TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded": 14.53, - "TestEmbeddedDepRekeyConvergesRenameChain": 14.34, - "TestEmbeddedDepRekeyMergesFreshDuplicateEdges": 29.13, - "TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies": 15.17, - "TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge": 14.62, - "TestEmbeddedDependencyEditorAcceptsADiamond": 3.43, - "TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget": 1.1, - "TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants": 1.53, - "TestEmbeddedDependencyEditorAcceptsAnExternalTarget": 0.93, - "TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes": 3.07, - "TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb": 3.21, - "TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder": 2.1, - "TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges": 2.3, - "TestEmbeddedDependencyEditorAttributesItsEventsToTheActor": 2.62, - "TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate": 5.12, - "TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType": 2.82, - "TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes": 6.74, - "TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes": 1.92, - "TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes": 1.75, - "TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest": 1.62, - "TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest": 1.83, - "TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest": 3.76, - "TestEmbeddedDependencyEditorRefusalWritesNothing": 2.44, - "TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop": 9.58, - "TestEmbeddedDependencyEditorRefusesAGhostSource": 1.87, - "TestEmbeddedDependencyEditorRefusesAMissingLocalTarget": 1.41, - "TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle": 3.51, - "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy": 2.37, - "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy": 2.44, - "TestEmbeddedDependencyEditorRefusesCrossPlaneCycle": 1.64, - "TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped": 1.18, - "TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked": 3.62, - "TestEmbeddedDependencyEditorRemoveIsIdempotent": 3.04, - "TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants": 6.72, - "TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge": 1.88, - "TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse": 2.47, - "TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge": 1.63, - "TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane": 1.47, - "TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane": 2.85, - "TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent": 2.06, - "TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion": 2.15, - "TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp": 3.79, - "TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe": 1.77, - "TestEmbeddedDependencyEditorSnapshotsTheRequest": 1.19, - "TestEmbeddedDependencyEditorValidationRefusalsWriteNothing": 1.01, - "TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove": 3.61, - "TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn": 3.14, - "TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers": 25.82, + "TestDualWriteContract": 1.46, + "TestDualWriteContractFlagOff": 0.46, + "TestDualWriteFixtureKitIsWired": 0.21, + "TestEdgeReaderContract": 6.2, + "TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing": 12.17, + "TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory": 0.89, + "TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes": 10.96, + "TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure": 11.11, + "TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed": 1.48, + "TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed": 0.52, + "TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty": 6.01, + "TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch": 13.75, + "TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest": 5.79, + "TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex": 12.23, + "TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp": 6.46, + "TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy": 9.44, + "TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess": 3.06, + "TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape": 12.34, + "TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex": 12.61, + "TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit": 13.57, + "TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded": 6.56, + "TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing": 11.7, + "TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed": 13.28, + "TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim": 1.27, + "TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges": 1.24, + "TestEmbeddedBlockedRecheckFailureKeepsTheWrite": 11.59, + "TestEmbeddedBlockedRecheckRecordsUnblockingWrites": 14.75, + "TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded": 12.69, + "TestEmbeddedCheckedWrappersRouteTheirPreconditions": 3.17, + "TestEmbeddedClaimIssueWrapperClaimsTheWispPlane": 0.79, + "TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne": 1.13, + "TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow": 0.8, + "TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone": 1.36, + "TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting": 0.72, + "TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting": 0.92, + "TestEmbeddedClaimerRefusalMessagesCarryTheirFragments": 1.02, + "TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt": 0.82, + "TestEmbeddedClaimerRefusesAWispIDAsNotFound": 0.58, + "TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState": 0.64, + "TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween": 1.11, + "TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder": 1.42, + "TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds": 0.69, + "TestEmbeddedCommitToleratesNothingToCommit": 0.53, + "TestEmbeddedConfigCommitStagesConfig": 0.33, + "TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded": 8.79, + "TestEmbeddedDepRekeyConvergesRenameChain": 8.79, + "TestEmbeddedDepRekeyMergesFreshDuplicateEdges": 15.36, + "TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies": 7.66, + "TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge": 8.19, + "TestEmbeddedDependencyEditorAcceptsADiamond": 3.03, + "TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget": 0.6, + "TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants": 1.26, + "TestEmbeddedDependencyEditorAcceptsAnExternalTarget": 0.59, + "TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes": 2.3, + "TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb": 1.9, + "TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder": 1.85, + "TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges": 2.17, + "TestEmbeddedDependencyEditorAttributesItsEventsToTheActor": 1.82, + "TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate": 4.26, + "TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType": 1.91, + "TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes": 4.53, + "TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes": 1.35, + "TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes": 1.11, + "TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest": 0.69, + "TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest": 1.0, + "TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest": 2.4, + "TestEmbeddedDependencyEditorRefusalWritesNothing": 1.38, + "TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop": 5.9, + "TestEmbeddedDependencyEditorRefusesAGhostSource": 0.66, + "TestEmbeddedDependencyEditorRefusesAMissingLocalTarget": 1.15, + "TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle": 2.52, + "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy": 2.06, + "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy": 1.78, + "TestEmbeddedDependencyEditorRefusesCrossPlaneCycle": 0.98, + "TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped": 0.49, + "TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked": 2.53, + "TestEmbeddedDependencyEditorRemoveIsIdempotent": 2.13, + "TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants": 4.83, + "TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge": 1.19, + "TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse": 1.57, + "TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge": 0.92, + "TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane": 0.78, + "TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane": 1.61, + "TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent": 1.23, + "TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion": 1.36, + "TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion": 1.6, + "TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp": 2.17, + "TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe": 0.98, + "TestEmbeddedDependencyEditorSnapshotsTheRequest": 0.73, + "TestEmbeddedDependencyEditorValidationRefusalsWriteNothing": 0.72, + "TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove": 2.29, + "TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn": 1.89, + "TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers": 15.34, "TestEmbeddedDoltStoreIssueLifecycleBuildsOperations": 0.0, "TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore": 0.0, - "TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion": 5.86, - "TestEmbeddedGetMergeBlockersOnAQuietStore": 0.52, - "TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches": 2.24, - "TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones": 12.11, - "TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace": 15.91, - "TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables": 13.02, - "TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape": 16.07, - "TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape": 31.85, - "TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone": 4.09, - "TestEmbeddedIssueOperationsCloseAndReopenAWisp": 1.72, - "TestEmbeddedIssueOperationsCreateAggregatesRelations": 2.22, - "TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp": 2.84, - "TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies": 3.31, - "TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure": 3.64, - "TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables": 2.25, - "TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps": 4.22, - "TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID": 5.14, - "TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction": 4.32, - "TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults": 5.07, - "TestEmbeddedIssueOperationsRequestValuesAreNotMutated": 2.9, - "TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged": 15.17, - "TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix": 2.6, - "TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState": 4.89, - "TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses": 2.93, - "TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim": 1.03, - "TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose": 17.14, - "TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent": 2.85, - "TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel": 0.73, - "TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps": 2.44, - "TestEmbeddedIssueOperationsUpdateLabelPatchOrdering": 3.02, - "TestEmbeddedIssueOperationsUpdateLabelPatchValueRules": 2.64, - "TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates": 2.62, - "TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel": 0.76, - "TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure": 5.78, - "TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes": 1.5, - "TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary": 2.34, - "TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress": 1.77, - "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender": 7.63, - "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers": 6.92, - "TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField": 1.74, - "TestEmbeddedLenientOpenWarning_DataBehind": 73.08, - "TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance": 17.96, - "TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge": 0.76, - "TestEmbeddedMergeAndSettleFKCascadeRepair": 2.52, - "TestEmbeddedMergeAndSettleMemoryConfigConflict": 0.68, - "TestEmbeddedMergeAndSettleMetadataConflict": 0.74, - "TestEmbeddedMergeAndSettleRefusesUnknownTable": 0.91, - "TestEmbeddedMergeAndSettleReportsOperatorConflicts": 1.14, - "TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict": 0.82, - "TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566": 31.96, - "TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690": 15.34, - "TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566": 15.45, - "TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566": 15.77, - "TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain": 15.59, - "TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext": 20.01, - "TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation": 9.28, - "TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints": 10.02, - "TestEmbeddedMigration0058HealsAlreadyAffectedDatabase": 10.23, - "TestEmbeddedMigration0062FlipsEventsToIgnoredPlane": 8.38, - "TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke": 17.42, - "TestEmbeddedOpenForReadOnlyCommand_LenientGate": 18.99, - "TestEmbeddedOpenForRemoteSync_DataBehind": 35.97, - "TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal": 18.37, + "TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion": 3.83, + "TestEmbeddedGetMergeBlockersOnAQuietStore": 0.3, + "TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches": 2.06, + "TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones": 8.68, + "TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace": 9.18, + "TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables": 6.47, + "TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape": 7.57, + "TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape": 29.07, + "TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone": 2.19, + "TestEmbeddedIssueOperationsCloseAndReopenAWisp": 1.33, + "TestEmbeddedIssueOperationsCreateAggregatesRelations": 1.42, + "TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp": 1.34, + "TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies": 1.56, + "TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure": 2.22, + "TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables": 1.16, + "TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps": 1.73, + "TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID": 3.79, + "TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction": 3.06, + "TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults": 3.54, + "TestEmbeddedIssueOperationsRequestValuesAreNotMutated": 2.19, + "TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged": 9.04, + "TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix": 2.15, + "TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState": 3.17, + "TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses": 2.12, + "TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim": 0.65, + "TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose": 9.43, + "TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent": 1.36, + "TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel": 0.52, + "TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps": 0.98, + "TestEmbeddedIssueOperationsUpdateLabelPatchOrdering": 1.06, + "TestEmbeddedIssueOperationsUpdateLabelPatchValueRules": 1.24, + "TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates": 1.93, + "TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel": 0.65, + "TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure": 6.98, + "TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes": 0.77, + "TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary": 1.62, + "TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress": 1.03, + "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender": 5.16, + "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers": 3.93, + "TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField": 1.6, + "TestEmbeddedLenientOpenWarning_DataBehind": 55.67, + "TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance": 10.91, + "TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge": 0.45, + "TestEmbeddedMergeAndSettleFKCascadeRepair": 1.89, + "TestEmbeddedMergeAndSettleMemoryConfigConflict": 0.47, + "TestEmbeddedMergeAndSettleMetadataConflict": 0.45, + "TestEmbeddedMergeAndSettleRefusesUnknownTable": 0.8, + "TestEmbeddedMergeAndSettleReportsOperatorConflicts": 0.69, + "TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict": 0.49, + "TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers": 14.45, + "TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566": 18.94, + "TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690": 13.55, + "TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566": 7.62, + "TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566": 8.24, + "TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain": 10.21, + "TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext": 10.7, + "TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation": 6.17, + "TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints": 7.11, + "TestEmbeddedMigration0058HealsAlreadyAffectedDatabase": 6.63, + "TestEmbeddedMigration0062FlipsEventsToIgnoredPlane": 5.08, + "TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke": 11.27, + "TestEmbeddedOpenForReadOnlyCommand_LenientGate": 10.66, + "TestEmbeddedOpenForRemoteSync_DataBehind": 21.05, + "TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal": 10.86, "TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites": 0.0, - "TestEmbeddedOpenReadOnly_SkipsGateAndMigrations": 18.77, - "TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape": 57.06, - "TestEmbeddedReaderDoesNotMutateTheCallerRequest": 0.91, - "TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows": 1.84, - "TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue": 1.46, - "TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay": 0.48, - "TestEmbeddedReaderGetOptionalRowListsAreOffByDefault": 2.13, - "TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes": 1.81, - "TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse": 2.8, - "TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge": 5.14, - "TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides": 7.58, - "TestEmbeddedReaderListEmptyPageIsWellFormed": 0.79, - "TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder": 6.4, - "TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates": 3.76, - "TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder": 1.56, - "TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing": 4.24, - "TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress": 2.66, - "TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset": 4.12, - "TestEmbeddedReaderListMaxRowsIsHonored": 2.77, - "TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch": 1.84, - "TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn": 2.03, - "TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing": 4.86, - "TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet": 2.64, - "TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters": 2.71, - "TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry": 2.87, - "TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary": 1.04, - "TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse": 2.97, - "TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet": 1.89, - "TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt": 2.31, - "TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage": 3.16, - "TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse": 1.62, - "TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType": 1.92, - "TestEmbeddedReaderReadyDeferredAndEphemeralGates": 1.87, - "TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary": 5.32, - "TestEmbeddedReaderReadyLimitBoundary": 1.97, - "TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded": 6.4, - "TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix": 71.65, - "TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants": 3.93, - "TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions": 3.2, - "TestEmbeddedReaderReadySortPoliciesOrderTheSameRows": 1.64, - "TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists": 1.82, - "TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits": 1.62, - "TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState": 2.4, - "TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest": 0.64, - "TestEmbeddedReadyClaimerEmptyFrontIsNormal": 1.2, - "TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven": 5.45, - "TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities": 4.27, - "TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne": 2.09, - "TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet": 1.15, - "TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin": 1.45, - "TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor": 0.89, - "TestEmbeddedReadyClaimerSkipsIneligibleFrontRows": 1.83, - "TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped": 0.72, - "TestEmbeddedRecomputeAllBlockedWiring": 1.9, - "TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection": 2.24, - "TestEmbeddedRelationsAnswersInThePinnedOrder": 2.28, - "TestEmbeddedRelationsDirectionSelectsTheInverseGraph": 1.71, - "TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary": 2.03, - "TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer": 1.81, - "TestEmbeddedRelationsLeavesTheCallersRequestAlone": 1.41, - "TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether": 2.73, - "TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength": 0.59, - "TestEmbeddedRelationsRefusesAnUnusableTypeFilter": 0.54, - "TestEmbeddedRelationsRefusesTheZeroDirection": 0.59, - "TestEmbeddedRelationsResolvesAWispAnchor": 1.56, - "TestEmbeddedRelationsResolvesTheAnchorIDExactly": 1.45, - "TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue": 0.72, - "TestEmbeddedRemoteMigrateGate_BlocksReopen": 18.03, - "TestEmbeddedRenameKeepsDependencyIDsDeterministic": 13.78, - "TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing": 1.36, - "TestEmbeddedReopenIssueCategorySemantics": 5.51, - "TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit": 0.81, - "TestEmbeddedRoleFixtureKitHooksAreUsable": 1.85, + "TestEmbeddedOpenReadOnly_SkipsGateAndMigrations": 12.74, + "TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape": 36.7, + "TestEmbeddedReaderDoesNotMutateTheCallerRequest": 0.57, + "TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows": 1.08, + "TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue": 1.0, + "TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay": 0.32, + "TestEmbeddedReaderGetOptionalRowListsAreOffByDefault": 1.44, + "TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes": 1.46, + "TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse": 1.95, + "TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge": 3.36, + "TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides": 3.68, + "TestEmbeddedReaderListEmptyPageIsWellFormed": 0.46, + "TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder": 3.81, + "TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates": 2.31, + "TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder": 0.92, + "TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing": 3.0, + "TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress": 1.82, + "TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset": 2.54, + "TestEmbeddedReaderListMaxRowsIsHonored": 1.65, + "TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch": 2.14, + "TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn": 1.26, + "TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing": 2.44, + "TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet": 1.31, + "TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters": 1.83, + "TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry": 1.79, + "TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary": 0.7, + "TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse": 2.25, + "TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet": 1.23, + "TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt": 1.58, + "TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage": 2.27, + "TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse": 1.09, + "TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType": 0.95, + "TestEmbeddedReaderReadyDeferredAndEphemeralGates": 1.2, + "TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary": 2.49, + "TestEmbeddedReaderReadyLimitBoundary": 1.18, + "TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded": 4.48, + "TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix": 44.08, + "TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants": 3.71, + "TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions": 1.77, + "TestEmbeddedReaderReadySortPoliciesOrderTheSameRows": 0.53, + "TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists": 1.05, + "TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits": 0.95, + "TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState": 1.57, + "TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest": 0.34, + "TestEmbeddedReadyClaimerEmptyFrontIsNormal": 0.72, + "TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven": 3.13, + "TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities": 3.83, + "TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne": 1.87, + "TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet": 0.73, + "TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin": 0.78, + "TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor": 0.46, + "TestEmbeddedReadyClaimerSkipsIneligibleFrontRows": 1.04, + "TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped": 5.52, + "TestEmbeddedRecomputeAllBlockedWiring": 1.79, + "TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection": 1.54, + "TestEmbeddedRelationsAnswersInThePinnedOrder": 1.21, + "TestEmbeddedRelationsDirectionSelectsTheInverseGraph": 1.0, + "TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary": 0.8, + "TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer": 1.16, + "TestEmbeddedRelationsLeavesTheCallersRequestAlone": 0.81, + "TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether": 1.47, + "TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength": 0.4, + "TestEmbeddedRelationsRefusesAnUnusableTypeFilter": 0.42, + "TestEmbeddedRelationsRefusesTheZeroDirection": 0.46, + "TestEmbeddedRelationsResolvesAWispAnchor": 1.03, + "TestEmbeddedRelationsResolvesTheAnchorIDExactly": 1.09, + "TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue": 0.42, + "TestEmbeddedRemoteMigrateGate_BlocksReopen": 9.98, + "TestEmbeddedRenameKeepsDependencyIDsDeterministic": 8.71, + "TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing": 0.94, + "TestEmbeddedReopenIssueCategorySemantics": 3.54, + "TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit": 0.62, + "TestEmbeddedRoleFixtureKitHooksAreUsable": 1.35, "TestEmbeddedSQLCommitResponseLossIsIndeterminate": 0.0, - "TestEmbeddedSyncCommitsConfigBeforeMerge": 1.4, - "TestEmbeddedTempRebuildIntermediatesNeverReachHistory": 21.52, - "TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges": 1.63, - "TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit": 3.09, - "TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges": 2.64, - "TestEmbeddedTransactionWispLifecyclePersistsDurableDependent": 8.69, + "TestEmbeddedSyncCommitsConfigBeforeMerge": 0.7, + "TestEmbeddedTempRebuildIntermediatesNeverReachHistory": 10.07, + "TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges": 0.86, + "TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit": 2.33, + "TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges": 1.52, + "TestEmbeddedTransactionWispLifecyclePersistsDurableDependent": 6.16, "TestEpochContract": 0.0, - "TestEventsAutoPruneBoundsTheJournalToTheFloors": 2.74, - "TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded": 1.51, - "TestEventsAutoPruneThrottleSkipsTheSecondPass": 3.34, - "TestEventsJournalAccessor": 2.42, - "TestEventsJournalPruneNeverSplitsASkewedPair": 2.51, - "TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff": 2.02, - "TestEventsJournalReadRefusesAnInteriorHole": 1.83, - "TestEventsJournalRestartAndRetentionBoundary": 4.32, - "TestEventsJournal_ActorMatrixOnEmbedded": 5.04, - "TestEventsJournal_EmbeddedConcurrentGaplessNoDup": 4.15, - "TestEventsSinceEmbedded": 2.2, + "TestEventsAutoPruneBoundsTheJournalToTheFloors": 1.73, + "TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded": 0.97, + "TestEventsAutoPruneThrottleSkipsTheSecondPass": 1.57, + "TestEventsJournalAccessor": 1.66, + "TestEventsJournalPruneNeverSplitsASkewedPair": 1.48, + "TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff": 0.97, + "TestEventsJournalReadRefusesAnInteriorHole": 1.14, + "TestEventsJournalRestartAndRetentionBoundary": 1.46, + "TestEventsJournal_ActorMatrixOnEmbedded": 4.21, + "TestEventsJournal_EmbeddedConcurrentGaplessNoDup": 2.6, + "TestEventsSinceEmbedded": 0.98, "TestExpectedRevisionContract": 0.0, - "TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree": 38.8, - "TestForeignSkipDetailIsBounded": 9.16, - "TestGetDependentRecordsEmbedded": 2.76, - "TestGetDependentRecordsForIssuesEmbedded": 4.19, - "TestGetIssue": 3.8, - "TestGetIssueCommentsPageEmbedded": 5.56, - "TestGetLabels": 1.66, - "TestGetNextChildID": 5.99, - "TestGetReadyWorkIncludeEphemeralAppliesWispFilters": 8.29, - "TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps": 3.58, - "TestGetReadyWorkMoleculeFilter": 2.41, + "TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree": 24.77, + "TestForeignSkipDetailIsBounded": 5.58, + "TestGetDependentRecordsEmbedded": 2.15, + "TestGetDependentRecordsForIssuesEmbedded": 3.4, + "TestGetIssue": 2.37, + "TestGetIssueCommentsPageEmbedded": 3.1, + "TestGetLabels": 1.25, + "TestGetNextChildID": 3.75, + "TestGetReadyWorkIncludeEphemeralAppliesWispFilters": 6.5, + "TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps": 1.58, + "TestGetReadyWorkMoleculeFilter": 1.61, "TestGitRefPinsMatchDolt": 0.0, - "TestGraphCounterContract": 15.92, + "TestGraphCounterContract": 9.86, + "TestGraphWalkPlansHonorJoinHints": 5.53, "TestHasRepository": 0.0, - "TestHeartbeatMatchesActorAcrossSpellingEmbedded": 1.01, - "TestHeartbeatRejectsWispEmbedded": 0.69, + "TestHeartbeatMatchesActorAcrossSpellingEmbedded": 0.56, + "TestHeartbeatRejectsWispEmbedded": 0.29, "TestHelperProcess": 0.0, - "TestHistoryPresenceContract": 3.26, - "TestHistory_NullTextColumns": 1.66, - "TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore": 3.06, - "TestIgnoredCursorHealConvergesFromEveryCrashState": 4.9, - "TestIgnoredCursorHealIsIdempotent": 2.0, - "TestIgnoredCursorHealNeverResurrectsRolledBackVersions": 1.95, - "TestIgnoredCursorHealRespectsOperatorOverride": 1.02, - "TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern": 0.95, - "TestIgnoredCursorHealResumeRespectsOperatorOverride": 1.14, - "TestIsBlockedBatchEmbedded": 2.64, - "TestJournalContract": 9.95, - "TestLargeBatchApplyStatementCounts356_Embedded": 79.51, + "TestHistoryPresenceContract": 1.96, + "TestHistory_NullTextColumns": 0.97, + "TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore": 1.86, + "TestIgnoredCursorHealConvergesFromEveryCrashState": 3.68, + "TestIgnoredCursorHealIsIdempotent": 1.62, + "TestIgnoredCursorHealNeverResurrectsRolledBackVersions": 0.92, + "TestIgnoredCursorHealRespectsOperatorOverride": 0.55, + "TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern": 0.44, + "TestIgnoredCursorHealResumeRespectsOperatorOverride": 0.7, + "TestIsBlockedBatchEmbedded": 2.12, + "TestJournalContract": 9.43, + "TestLargeBatchApplyStatementCounts356_Embedded": 22.91, "TestLargeBatchApplyStatementCounts712_Embedded": 1.0, - "TestLargeBatchApplyStatementCountsClassic40_Embedded": 8.73, - "TestLargeBatchApplyWallClock_Embedded": 80.7, - "TestLeaseLifecycleEmbedded": 3.76, - "TestLegacyTrackedIgnoredCursorHealsOnOpen": 1.39, - "TestLegacyTrackedIgnoredCursorUnwedgesPull": 7.0, + "TestLargeBatchApplyStatementCountsClassic40_Embedded": 5.79, + "TestLargeBatchApplyWallClock_Embedded": 21.85, + "TestLargeGraphCreateTiming_Embedded": 0.0, + "TestLeaseLifecycleEmbedded": 3.01, + "TestLegacyTrackedIgnoredCursorHealsOnOpen": 0.83, + "TestLegacyTrackedIgnoredCursorUnwedgesPull": 4.25, "TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance": 0.0, "TestLenientGateWarningBody_ShapedDecisions": 0.0, "TestLenientGateWarning_ComposedOutput": 0.0, - "TestLifecycleCloseReopenContract": 76.34, - "TestLifecycleCreateContract": 12.17, - "TestLifecycleUpdateContract": 39.87, - "TestListWisps": 1.63, - "TestMemoriesContract": 13.94, - "TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle": 1.32, - "TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict": 1.22, - "TestMergeWithStrategy_OursKeepsLocal": 1.19, - "TestMergeWithStrategy_TheirsResolvesAndCommits": 1.19, - "TestMerge_NoStrategyStillErrorsWithEscapeHatchHint": 1.13, - "TestMetadataCASContract": 14.63, - "TestOpenAfterCloseCreatesNewStore": 17.64, - "TestOpenDifferentDirsReturnsDifferentStores": 36.11, - "TestOpenReturnsCachedStore": 19.64, - "TestPartitionWispIDs": 1.2, - "TestPreviewStoreRefusesLocalMetadataWrite": 17.18, - "TestPristineEmbeddedDoltFixtureRelocatesPrefixes": 2.42, - "TestPromoteClearsNoHistory": 3.01, - "TestPromoteStillWorksWithCollisionGuard": 1.36, - "TestPruneEventsBatchDeletesTheLowestSeqsOnly": 1.97, - "TestPushCommitsPendingChangesFirst": 89.08, - "TestQuerierContract": 15.49, - "TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery": 1.38, - "TestReadyCounterContract": 8.16, - "TestReadyWorkPageTotal": 30.66, - "TestReadyWorkPageTotalFilterMatrix": 23.68, - "TestReadyWorkStatementBudget": 22.78, - "TestReadyWorkWithMissingDeferredParentTables": 5.52, - "TestReclaimAnyReplicaOverride": 0.89, - "TestReclaimExpiredLeaseSurvivesRestartEmbedded": 20.55, - "TestReclaimRefusesForeignReplicaLease": 1.28, - "TestReleaserContract": 8.65, + "TestLifecycleCloseReopenContract": 64.3, + "TestLifecycleCreateContract": 9.5, + "TestLifecycleUpdateContract": 32.78, + "TestListWisps": 1.0, + "TestMemoriesContract": 10.65, + "TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle": 1.06, + "TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict": 1.14, + "TestMergeWithStrategy_OursKeepsLocal": 0.65, + "TestMergeWithStrategy_TheirsResolvesAndCommits": 0.61, + "TestMerge_NoStrategyStillErrorsWithEscapeHatchHint": 0.52, + "TestMetadataCASContract": 9.64, + "TestOpenAfterCloseCreatesNewStore": 11.35, + "TestOpenDifferentDirsReturnsDifferentStores": 22.39, + "TestOpenReturnsCachedStore": 10.72, + "TestPartitionWispIDs": 0.75, + "TestPreviewStoreRefusesLocalMetadataWrite": 11.12, + "TestPristineEmbeddedDoltFixtureRelocatesPrefixes": 1.63, + "TestPromoteClearsNoHistory": 1.77, + "TestPromoteStillWorksWithCollisionGuard": 6.08, + "TestPruneEventsBatchDeletesTheLowestSeqsOnly": 1.23, + "TestPushCommitsPendingChangesFirst": 59.03, + "TestQuerierContract": 7.95, + "TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery": 0.63, + "TestReadyCounterContract": 4.93, + "TestReadyWorkPageTotal": 21.9, + "TestReadyWorkPageTotalFilterMatrix": 15.52, + "TestReadyWorkStatementBudget": 14.59, + "TestReadyWorkWithMissingDeferredParentTables": 4.54, + "TestReclaimAnyReplicaOverride": 0.71, + "TestReclaimExpiredLeaseSurvivesRestartEmbedded": 13.82, + "TestReclaimRefusesForeignReplicaLease": 0.74, + "TestReleaserContract": 5.0, "TestRemoteEntryPointsUseVersionControlOps": 0.0, - "TestRemoteGitDataRefOnEngine": 2.82, - "TestRemoteVerbPlainRemoteKeepsAmbientEnv": 17.91, - "TestRemoteVerbPlainRemoteWithoutAuthPresentsNone": 17.51, - "TestRemoteVerbsPresentStoredPeerCredentials": 20.79, - "TestRenameLabel": 3.42, - "TestRenameLabelMintsOneVersionPerTouchedBead": 4.17, - "TestReopenIssue": 1.7, + "TestRemoteGitDataRefOnEngine": 1.4, + "TestRemoteVerbPlainRemoteKeepsAmbientEnv": 10.97, + "TestRemoteVerbPlainRemoteWithoutAuthPresentsNone": 12.14, + "TestRemoteVerbsPresentStoredPeerCredentials": 12.78, + "TestRenameLabel": 1.98, + "TestRenameLabelMintsOneVersionPerTouchedBead": 2.45, + "TestReopenIssue": 1.09, "TestRetentionContract": 0.0, - "TestRunInTransactionPostCommitStageFailureIsIndeterminate": 0.62, - "TestSchemaAfterInit": 17.05, - "TestSearchWithMissingWispPlaneTables": 3.33, + "TestRunInTransactionPostCommitStageFailureIsIndeterminate": 0.44, + "TestSchemaAfterInit": 10.4, + "TestSearchWithMissingWispPlaneTables": 1.92, "TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed": 0.0, - "TestStatsReporterContract": 14.39, - "TestSweeperContract": 25.68, + "TestStatsReporterContract": 9.56, + "TestSweeperContract": 17.21, "TestTransactionCleanupAfterSQLCommitIsIndeterminate": 0.0, "TestTransactionCleanupBeforeSQLCommitRemainsDefinite": 0.0, - "TestTreeWalkerContract": 22.54, - "TestUnknownProvenanceLeaseStaysReclaimable": 1.83, - "TestUpdateIssueType": 0.83, - "TestUpdateMergeOps_DoltEngine": 0.92, - "TestUpdateMergeOps_DoltEngine_TwoIssues": 1.16, - "TestVersionReconcilerContract": 4.02, - "TestWithPeerAuth": 87.8, - "TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials": 20.0, - "TestWithPeerAuth_DivergedRemoteURLFailsClosed": 17.14, - "TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials": 19.57, - "TestWithPeerAuth_KeyMismatchFailsClosed": 16.51, - "TestWithPeerAuth_MissingRemoteFailsClosed": 17.76, - "TestWithPeerAuth_RestoresAbsentEnv": 17.9, - "TestWithPeerAuth_RestoresEnvWhenCallbackFails": 16.84, - "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword": 121.04, - "TestWorkspaceConfigContract": 7.42 + "TestTreeWalkerContract": 12.82, + "TestUnknownProvenanceLeaseStaysReclaimable": 1.21, + "TestUpdateIssueType": 0.51, + "TestUpdateMergeOps_DoltEngine": 0.49, + "TestUpdateMergeOps_DoltEngine_TwoIssues": 0.77, + "TestVersionReconcilerContract": 2.4, + "TestWithPeerAuth": 51.7, + "TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials": 12.63, + "TestWithPeerAuth_DivergedRemoteURLFailsClosed": 12.79, + "TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials": 8.72, + "TestWithPeerAuth_KeyMismatchFailsClosed": 11.39, + "TestWithPeerAuth_MissingRemoteFailsClosed": 10.01, + "TestWithPeerAuth_RestoresAbsentEnv": 11.38, + "TestWithPeerAuth_RestoresEnvWhenCallbackFails": 12.11, + "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword": 61.05, + "TestWorkspaceConfigContract": 5.37 } } diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go index bbc953180..2bb020b41 100644 --- a/scripts/shard_manifest_test.go +++ b/scripts/shard_manifest_test.go @@ -19,7 +19,7 @@ import ( // S3 (F1, mirroring F2's TestProxiedShardManifestGeneratorNotStale below — // see that test's doc comment for the full --check rationale, not repeated -// here): the Bazel-only 50-shard cmd block and 15-shard storage block are +// here): the Bazel-only 50-shard cmd block and 20-shard storage block are // not frozen like their files' legacy 20- and 5-shard blocks. // gen_embedded_{cmd,storage}_shard_manifest.py --check verifies only that // the committed block names every discovered test exactly once, failing with @@ -45,7 +45,7 @@ func TestCmdEmbeddedShardManifestGeneratorNotStale(t *testing.T) { // TestStorageEmbeddedShardManifestGeneratorNotStale mirrors // TestCmdEmbeddedShardManifestGeneratorNotStale above for the storage tier's -// Bazel-only 15-shard block; see that test's doc comment. +// Bazel-only 20-shard block; see that test's doc comment. func TestStorageEmbeddedShardManifestGeneratorNotStale(t *testing.T) { python := requireHostTool(t, "python3") root := sourceRepoRoot(t) @@ -138,7 +138,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { // duration balancing. totalsSet[bazelEmbeddedCmdShardCount(t)] = true case ".github/scripts/embedded-storage-test-shard.sh": - // F1: mirrors the cmd case above for the 15-shard storage block. + // F1: mirrors the cmd case above for the 20-shard storage block. totalsSet[bazelEmbeddedStorageShardCount(t)] = true case ".github/scripts/server-storage-test-shard.sh": totalsSet[bazelServerFullShardCount(t)] = true From f60de63a75e1db09aff3defca580e8b05ba92e4d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 08:45:05 +0000 Subject: [PATCH 013/151] ci(bazel): 8 vCPU client for bazel-test, 4 vCPU for the other Bazel lanes (#7422) Client-side loading and analysis is CPU-bound and gates every action in a remote lane. Analysis-only measurements on Blacksmith (ga-vnycm2.8, seeded runner cache, gazelle tools from the repository cache): bazel-test 25.6 s on 4 vCPU vs 16.3 s on 8; the other lanes 17-38 s on 2 vCPU vs 12-22 s on 4. rbe-prewarm runs no Bazel client and stays on 2 vCPU (owner decision, ga-vnycm2.12). Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 24 ++++++++++++------------ scripts/ci_blacksmith_runner_test.go | 28 ++++++++++++++++------------ scripts/ci_workflow_test.go | 10 +++++----- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index eb761aa7f..b3d13b0f6 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -729,14 +729,14 @@ jobs: # Fork modes (fork-ro/fork-rw) execute remotely but stay GitHub-hosted, # like every fork job: the runner is only a client. # - # 4 vCPU, not the other lanes' 2: with every action a remote cache hit, + # 8 vCPU, not the other lanes' 4: with every action a remote cache hit, # this lane's `bazel test //... --config=ci` spent ~63 of 81s in # client-side loading and analysis (1798 packages, 66k configured # targets; critical path 7.65s, merge group 37721729916), which Bazel - # parallelizes across the client's cores. Revert to 2 vCPU if the A/B - # does not save at least 15s on that step. + # parallelizes across the client's cores. Analysis-only, seeded runner + # cache (ga-vnycm2.8): 38.3s on 2 vCPU, 25.6s on 4, 16.3s on 8. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed # worker pool shared with gascity). Skipped only in mode skip (same-repo # runs while RBE_WEST_WORKERS is unset). @@ -1009,7 +1009,7 @@ jobs: name: pure-Go and js/wasm # Same runner and skip rule as bazel-test. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 outputs: @@ -1226,7 +1226,7 @@ jobs: bazel-release-cross: name: release cross-compile needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 outputs: @@ -1316,7 +1316,7 @@ jobs: # The runs-on expression is the shared one: Blacksmith in mode remote, # GitHub-hosted in the fork modes (a client only). needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 60 outputs: @@ -1486,7 +1486,7 @@ jobs: # A caller may turn it off with integration: "off"; pr.yml and # bazel-farm.yml pass "on". needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }} timeout-minutes: 85 outputs: @@ -1622,7 +1622,7 @@ jobs: name: dolt-server lane # Same runner and skip rule as bazel-test. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 outputs: @@ -1751,7 +1751,7 @@ jobs: # This lane is the tier's only PR run (pr-risk.yml's legacy jobs are # retired) and pr.yml's gate requires it to have run here and passed. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 30 outputs: @@ -1922,7 +1922,7 @@ jobs: # This lane is the tier's only PR run (pr-risk.yml's legacy jobs are # retired) and pr.yml's gate requires it to have run here and passed. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 30 outputs: @@ -2084,7 +2084,7 @@ jobs: bazel-cmd-dolt: name: cmd/bd Dolt-server tier needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }} timeout-minutes: 85 continue-on-error: true diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index 35685349a..d9b6e5fbc 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -63,17 +63,20 @@ func bazelRemoteRunsOn(label string) string { return "${{ needs.rbe.outputs.mode == 'remote' && '" + label + "' || 'ubuntu-latest' }}" } -// The default lane size: the runner is only an RBE client (every action -// executes on rbe-west), so 2 vCPU is enough. -var bazelLaneRunsOn = bazelRemoteRunsOn("blacksmith-2vcpu-ubuntu-2404") +// The default lane size: every action executes on rbe-west, but the +// client's loading and analysis is CPU-bound and gates every action +// (ga-vnycm2.8: 17-38 s on 2 vCPU, 12-26 s on 4), so 4 vCPU. +var bazelLaneRunsOn = bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404") -// Runner-size A/B (ci/bigger-runners-ab, 2026-10-08): bazel-test's -// `bazel test //... --config=ci` spent ~63 of its 81 s in client-side loading -// and analysis (1798 packages, 66k configured targets; critical path 7.65 s, -// every action a remote cache hit), which Skyframe parallelizes across the -// client's cores. 4 vCPU; revert to bazelLaneRunsOn if the PR run does not -// save at least 15 s on that step. -var bazelTestLaneRunsOn = bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404") +// rbe-prewarm only dispatches a pool worker: no Bazel client, 2 vCPU. +var bazelPrewarmRunsOn = bazelRemoteRunsOn("blacksmith-2vcpu-ubuntu-2404") + +// bazel-test's `bazel test //... --config=ci` spent ~63 of its 81 s in +// client-side loading and analysis (1798 packages, 66k configured targets; +// critical path 7.65 s, every action a remote cache hit), which Skyframe +// parallelizes across the client's cores: 8 vCPU (ga-vnycm2.8 analysis-only: +// 25.6 s on 4 vCPU, 16.3 s on 8). +var bazelTestLaneRunsOn = bazelRemoteRunsOn("blacksmith-8vcpu-ubuntu-2404") // The package gates: package-npm at 4 vCPU (F3); package-mcp at 8 vCPU with // pytest -n 16 (bazelMCPPytestWorkers), since its pytest run is dominated by @@ -565,9 +568,10 @@ func TestBlacksmithWindowsMacOSRunsOnEveryEvent(t *testing.T) { func TestBlacksmithBazelRunnerSizes(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) wantRemote := map[string]string{ - bazelJobName: "blacksmith-4vcpu-ubuntu-2404", + bazelJobName: "blacksmith-8vcpu-ubuntu-2404", bazelPackageMCPJobName: "blacksmith-8vcpu-ubuntu-2404", bazelPackageNPMJobName: "blacksmith-4vcpu-ubuntu-2404", + bazelRBEPrewarmJobName: "blacksmith-2vcpu-ubuntu-2404", } mcpGate := workflow.job(t, bazelPackageMCPJobName).step(t, "Run MCP package gate") if got := mcpGate.Env["BEADS_MCP_PYTEST_WORKERS"]; got != bazelMCPPytestWorkers { @@ -581,7 +585,7 @@ func TestBlacksmithBazelRunnerSizes(t *testing.T) { } want := "ubuntu-latest" if mode == "remote" { - want = "blacksmith-2vcpu-ubuntu-2404" + want = "blacksmith-4vcpu-ubuntu-2404" if label, ok := wantRemote[name]; ok { want = label } diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 174e8255e..68c696a03 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1941,8 +1941,8 @@ const bazelIntegIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outp const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}" // F3: the package gates' runners (bazelPackageRunsOn, in -// ci_blacksmith_runner_test.go with the other runner sizes) are larger than -// bazel.yml's usual 2 vCPU because they run pytest/npm on the runner itself. +// ci_blacksmith_runner_test.go with the other runner sizes) are sized for +// pytest/npm on the runner itself, not for a Bazel client alone. // Mode remote only: the gates take no rbe-fork certificate, so fork modes // (enabled too) build bd with go build on a GitHub-hosted runner, as before. @@ -2164,11 +2164,11 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { // lane does: its build is --config=integration's. wantJobIf = bazelIntegIf } else if name == bazelRBEPrewarmJobName { - // Same runs-on ternary as every other lane (wantRunsOn, set - // above); only the if differs. Not a lane: never uses + // The lanes' runs-on ternary at 2 vCPU (no Bazel client); only + // the size and the if differ. Not a lane: never uses // setup-bazel, so wantJobSetupEnv is moot (the per-step check // below only fires for a setup-bazel step). - wantJobIf = bazelRBEPrewarmIf + wantJobRunsOn, wantJobIf = bazelPrewarmRunsOn, bazelRBEPrewarmIf } if job.RunsOn != wantJobRunsOn { t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantJobRunsOn) From c4c66d42a2e11cea3254446afd5d13dde7478633 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 08:53:35 +0000 Subject: [PATCH 014/151] ci(bazel): rebalance the proxied-server tier on rbe-west measurements (30 -> 44 shards) (#7418) //cmd/bd:bd_proxied_test's 30-shard block was packed from a local capture of each test's , which for a t.Parallel test spans the time it sat paused behind its shard's other tests. Cheap tests that waited looked as heavy as the shard, so the "balanced" block ran 38-286s per shard in CI (356s on rbe-west). - scripts/ci/proxied_test_durations.json: re-measured on rbe-west (two --nocache_test_results runs), each test's cost the sum of its leaf tests' , averaged over both runs. - Measured where a test's time goes: nearly all of it is `bd init --proxied-server` against the shard's dolt sql-server (~2,000 statements, 81 DOLT_COMMITs, ~6-12 server CPU-seconds per init), one per subtest. Top-level tests with ~20 such subtests ran up to ~350s alone, above any shard budget, so the fifteen heaviest are split into two or three top-level functions (TestProxiedServerUpdate/-UpdateB/-UpdateC, ...). Each subtest is moved verbatim; the split functions repeat the original's preamble. - 30 -> 44 shards, LPT-packed to <=430 cost-seconds per shard. rbe-west, --nocache_test_results: before 30 shards, max 356s, 5,671 shard-seconds; after 44 shards, max 159s (median 138s), 5,883 shard-seconds. Bead: ga-vnycm2.3 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/proxied-cmd-test-shards.txt | 413 ++++++++------- .github/workflows/bazel.yml | 44 +- cmd/bd/BUILD.bazel | 6 +- cmd/bd/close_proxied_integration_test.go | 17 +- cmd/bd/config_proxied_integration_test.go | 8 + cmd/bd/create_proxied_integration_test.go | 16 + cmd/bd/dep_proxied_integration_test.go | 32 +- cmd/bd/gate_check_proxied_integration_test.go | 11 + cmd/bd/label_proxied_integration_test.go | 18 + cmd/bd/ready_proxied_integration_test.go | 17 +- cmd/bd/reopen_proxied_integration_test.go | 8 + cmd/bd/show_proxied_integration_test.go | 26 +- cmd/bd/update_proxied_integration_test.go | 34 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 2 +- engdocs/TESTING.md | 4 +- scripts/ci/gen_proxied_shard_manifest.py | 2 +- scripts/ci/proxied_test_durations.json | 480 +++++++++--------- scripts/shard_manifest_test.go | 7 +- 18 files changed, 669 insertions(+), 476 deletions(-) diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index b51c393b8..195ff2c5f 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -225,12 +225,12 @@ # # Format: # -# 30-shard split for the Bazel-only proxied-server tier +# 44-shard split for the Bazel-only proxied-server tier # (bazel-proxied in .github/workflows/bazel.yml), bin-packed # longest-processing-time-first by measured wall-time from # scripts/ci/proxied_test_durations.json (see that file for # provenance and its "relative weight, not absolute SLA" caveat). -# Regenerate with scripts/ci/gen_proxied_shard_manifest.py 30 +# Regenerate with scripts/ci/gen_proxied_shard_manifest.py 44 # --weights=duration --write after adding or removing # TestProxiedServer*/TestServerMode* functions. --write is incremental: # it keeps every already-assigned test on its current shard and only @@ -252,219 +252,254 @@ # regeneration. Add them to the generator or the durations file # instead. -30 1 TestProxiedServerGateList -30 1 TestProxiedServerHuman -30 1 TestProxiedServerReclaim -30 1 TestProxiedServerSQLStatementRouting -30 1 TestProxiedServerServeLifecycle +44 1 TestProxiedServerDep2 +44 1 TestProxiedServerServeAddDependencies -30 2 TestProxiedServerCloseBeadGate -30 2 TestProxiedServerDelete -30 2 TestProxiedServerDeleteForceOrphans -30 2 TestProxiedServerIfRevisionGuardAssign -30 2 TestProxiedServerListComments -30 2 TestProxiedServerPromote -30 2 TestProxiedServerTypesAgreeWithCreate +44 2 TestProxiedServerClose2 +44 2 TestProxiedServerCount +44 2 TestProxiedServerEventsJournalOffByDefault +44 2 TestProxiedServerServeRefusesAStreamOnADisabledJournal -30 3 TestProxiedServerDuplicates -30 3 TestProxiedServerReadyWisp -30 3 TestProxiedServerServeDelete -30 3 TestProxiedServerServeModeGate -30 3 TestProxiedServerServeReadRevision +44 3 TestProxiedServerEventsJournal +44 3 TestProxiedServerServeReadsTheJournalItJustWrote +44 3 TestProxiedServerUpdate3 +44 3 TestServerModeSharedServeGlobalReportsTheServedDatabase -30 4 TestProxiedServerGateCreateInheritsRepoMetadata -30 4 TestProxiedServerMolBurn -30 4 TestProxiedServerReadyConcurrent -30 4 TestProxiedServerServeAddDependencies -30 4 TestProxiedServerServeRelatedIssues +44 4 TestProxiedServerClose2B +44 4 TestProxiedServerOutageReconnectAcceptanceMatrix +44 4 TestProxiedServerServeDelete +44 4 TestProxiedServerSetState -30 5 TestProxiedServerCleanDatabases -30 5 TestProxiedServerCreate -30 5 TestProxiedServerEventsJournal -30 5 TestProxiedServerPour -30 5 TestProxiedServerServeUpdateClaim -30 5 TestServerModeDeferAutoWake +44 5 TestProxiedServerCreateInfraTypeRoutesToWisps +44 5 TestProxiedServerHuman +44 5 TestProxiedServerListComments +44 5 TestProxiedServerServeClaimNext -30 6 TestProxiedServerBatchParity -30 6 TestProxiedServerHistoryRemoteSupportedFrontDoorParity -30 6 TestProxiedServerMolShow -30 6 TestProxiedServerMultiStatementSQL -30 6 TestProxiedServerReady -30 6 TestProxiedServerServeRemoveDependency +44 6 TestProxiedServerAssign +44 6 TestProxiedServerCompactDolt +44 6 TestProxiedServerHistory +44 6 TestProxiedServerIfRevisionGuardDelete +44 6 TestProxiedServerKV +44 6 TestProxiedServerServeStreamsTheJournalItIsWriting -30 7 TestProxiedServerCleanDatabasesPurgeDropped -30 7 TestProxiedServerCount -30 7 TestProxiedServerCountIncludeInfra -30 7 TestProxiedServerDoltLifecycle -30 7 TestProxiedServerGC -30 7 TestProxiedServerUpdate +44 7 TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce +44 7 TestProxiedServerDoltLifecycle +44 7 TestProxiedServerGateFrontDoorDirectServerProxyParity +44 7 TestProxiedServerIfRevisionGuardClose +44 7 TestProxiedServerMolDistill +44 7 TestProxiedServerPathHelpers +44 7 TestProxiedServerStorageClass +44 7 TestServerModeServe -30 8 TestProxiedServerInitIfMissing -30 8 TestProxiedServerPathHelpers -30 8 TestProxiedServerReopenConcurrent -30 8 TestProxiedServerShow2 -30 8 TestProxiedServerTypes -30 8 TestProxiedServerUpdateClosePolicy -30 8 TestServerModeSharedServeGlobalReportsTheServedDatabase +44 8 TestProxiedServerDelete +44 8 TestProxiedServerPourRejectsEnumViolatingVar +44 8 TestProxiedServerSQL +44 8 TestProxiedServerTag +44 8 TestProxiedServerUpdate2B -30 9 TestProxiedServerCloseConcurrent -30 9 TestProxiedServerDep2 -30 9 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows -30 9 TestProxiedServerMolReadyGated -30 9 TestProxiedServerSQL +44 9 TestProxiedServerConfigSetMany +44 9 TestProxiedServerReopen +44 9 TestProxiedServerServeClaim +44 9 TestProxiedServerUnclaimIfAssigneeParity +44 9 TestProxiedServerUpdateConcurrentAppendNotes -30 10 TestProxiedServerAutoPrune -30 10 TestProxiedServerClose2 -30 10 TestProxiedServerPing -30 10 TestProxiedServerServeClose -30 10 TestProxiedServerServeEdgeCounts +44 10 TestProxiedServerDeleteBlockedJSONSingleDoc +44 10 TestProxiedServerLink +44 10 TestProxiedServerPriority +44 10 TestProxiedServerReadyWisp +44 10 TestProxiedServerServeAddComment -30 11 TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce -30 11 TestProxiedServerOutageReconnectAcceptanceMatrix -30 11 TestProxiedServerReady2 -30 11 TestProxiedServerServeReadsTheJournalItJustWrote -30 11 TestProxiedServerServeRefusesTheJournalWhenItIsDisabled -30 11 TestProxiedServerShow +44 11 TestProxiedServerAutoPrune +44 11 TestProxiedServerBatchParity +44 11 TestProxiedServerContext +44 11 TestProxiedServerCreate2 +44 11 TestProxiedServerExternalMutationPolicy -30 12 TestProxiedServerAssign -30 12 TestProxiedServerConfigSetMany -30 12 TestProxiedServerContext -30 12 TestProxiedServerInitExplicitBeadsDirIsAuthoritative -30 12 TestProxiedServerReadyClaimConcurrent -30 12 TestProxiedServerServeBatchClose +44 12 TestProxiedServerExternalCreate +44 12 TestProxiedServerServeRelatedIssues +44 12 TestProxiedServerUpdate +44 12 TestProxiedServerWispList +44 12 TestServerModeTypesAgreeWithCreate -30 13 TestProxiedServerChildren -30 13 TestProxiedServerConfig -30 13 TestProxiedServerDoltRemoteRemove -30 13 TestProxiedServerExternalMutationPolicy -30 13 TestProxiedServerPriority -30 13 TestProxiedServerServeUpdate +44 13 TestProxiedServerCreate +44 13 TestProxiedServerRecomputeBlocked +44 13 TestProxiedServerServeUpdate +44 13 TestProxiedServerShow2 +44 13 TestProxiedServerStale -30 14 TestProxiedServerInfo -30 14 TestProxiedServerLint -30 14 TestProxiedServerNote -30 14 TestProxiedServerQuery -30 14 TestProxiedServerUpdate3 -30 14 TestProxiedServerWispCreate +44 14 TestProxiedServerMolSquash +44 14 TestProxiedServerQuick +44 14 TestProxiedServerServeReadParity +44 14 TestProxiedServerServeRefusesAStaleCheckpointWithGone +44 14 TestProxiedServerShow2B -30 15 TestProxiedServerCompactDolt -30 15 TestProxiedServerMolDistill -30 15 TestProxiedServerMolProgress -30 15 TestProxiedServerServeClaimNext -30 15 TestProxiedServerServeRefusesAStreamOnADisabledJournal -30 15 TestProxiedServerUnclaimIfAssigneeParity +44 15 TestProxiedServerCountIncludeInfra +44 15 TestProxiedServerDeleteForceOrphans +44 15 TestProxiedServerDuplicates +44 15 TestProxiedServerLabel +44 15 TestProxiedServerServeBatchClose -30 16 TestProxiedServerCascadeParity -30 16 TestProxiedServerExternalCreate -30 16 TestProxiedServerGraph -30 16 TestProxiedServerLabel -30 16 TestProxiedServerSwarm -30 16 TestProxiedServerTodo -30 16 TestProxiedServerUpdateConcurrentMetadata +44 16 TestProxiedServerHistoryRemoteSupportedFrontDoorParity +44 16 TestProxiedServerQuery +44 16 TestProxiedServerSQLDatabaseFlag +44 16 TestProxiedServerShowB +44 16 TestProxiedServerWispCreate -30 17 TestProxiedServerClose -30 17 TestProxiedServerClose3 -30 17 TestProxiedServerClose4 -30 17 TestProxiedServerConfigConcurrent -30 17 TestProxiedServerDeleteConcurrent -30 17 TestProxiedServerIfRevisionGuardUpdate -30 17 TestProxiedServerSearch -30 17 TestProxiedServerServeReadParity -30 17 TestProxiedServerStatus +44 17 TestProxiedServerDoltRemoteRemove +44 17 TestProxiedServerImport +44 17 TestProxiedServerLint +44 17 TestProxiedServerServeEdgeCounts +44 17 TestProxiedServerStatus -30 18 TestProxiedServerHeartbeat -30 18 TestProxiedServerMolBond -30 18 TestProxiedServerSQLDatabaseFlag -30 18 TestProxiedServerServeCount -30 18 TestProxiedServerUpdate2 -30 18 TestServerModeTypesAgreeWithCreate +44 18 TestProxiedServerCloseConcurrent +44 18 TestProxiedServerDep2C +44 18 TestProxiedServerEdit +44 18 TestProxiedServerIfRevisionGuardUpdate +44 18 TestProxiedServerSearch -30 19 TestProxiedServerCreate2 -30 19 TestProxiedServerEdit -30 19 TestProxiedServerIfRevisionGuardReopen -30 19 TestProxiedServerPourRejectsEnumViolatingVar -30 19 TestProxiedServerReassignFence -30 19 TestProxiedServerUpdateWisp -30 19 TestServerModeServe +44 19 TestProxiedServerServeClose +44 19 TestProxiedServerTypesAgreeWithCreate +44 19 TestProxiedServerUpdate2 +44 19 TestProxiedServerUpdateClosePolicy -30 20 TestProxiedServerExternalPolicyDirectParity -30 20 TestProxiedServerHistory -30 20 TestProxiedServerImport -30 20 TestProxiedServerMemory -30 20 TestProxiedServerMolCurrent -30 20 TestProxiedServerServeStreamsTheJournalItIsWriting +44 20 TestProxiedServerDeferAutoWake +44 20 TestProxiedServerEpic +44 20 TestProxiedServerIfRevisionGuardReopen +44 20 TestProxiedServerInitIfMissing +44 20 TestProxiedServerReopenB +44 20 TestProxiedServerUpdateIfGuards -30 21 TestProxiedServerDeleteWisp -30 21 TestProxiedServerDepConcurrent -30 21 TestProxiedServerListSortRetiresTheWalk -30 21 TestProxiedServerPrime -30 21 TestProxiedServerRelate -30 21 TestProxiedServerSetState -30 21 TestProxiedServerUpdateConcurrentAppendNotes +44 21 TestProxiedServerBlocked +44 21 TestProxiedServerCleanDatabases +44 21 TestProxiedServerConfig +44 21 TestProxiedServerMolShow +44 21 TestProxiedServerPing +44 21 TestProxiedServerServeActivatesTheEventsJournal -30 22 TestProxiedServerCompactHistory -30 22 TestProxiedServerGateLifecycle -30 22 TestProxiedServerKV -30 22 TestProxiedServerShow3 -30 22 TestProxiedServerStale -30 22 TestProxiedServerStatuses +44 22 TestProxiedServerCloseBeadGate +44 22 TestProxiedServerExternalPolicyDirectParity +44 22 TestProxiedServerFindDuplicates +44 22 TestProxiedServerReadyB +44 22 TestProxiedServerServeCreate -30 23 TestProxiedServerBackupRefusedOnExternalTopology -30 23 TestProxiedServerCloseSiblingBlockersFanIn -30 23 TestProxiedServerDeleteBlockedJSONSingleDoc -30 23 TestProxiedServerQuick -30 23 TestProxiedServerStorageClass -30 23 TestProxiedServerUpdateHooks -30 23 TestProxiedServerWispGC +44 23 TestProxiedServerCloseSiblingBlockersFanIn +44 23 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity +44 23 TestProxiedServerPour +44 23 TestProxiedServerShow3 +44 23 TestProxiedServerUpdateConcurrentClaim -30 24 TestProxiedServerCreateInfraTypeRoutesToWisps -30 24 TestProxiedServerEpic -30 24 TestProxiedServerLink -30 24 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity -30 24 TestProxiedServerRecomputeBlocked -30 24 TestServerModeServeSkipsPostRunMaintenance +44 24 TestProxiedServerLabelC +44 24 TestProxiedServerMemory +44 24 TestProxiedServerMolStale +44 24 TestProxiedServerServeUpdateClaim +44 24 TestProxiedServerSwarm -30 25 TestProxiedServerDeferUndefer -30 25 TestProxiedServerOrphans -30 25 TestProxiedServerServeClaim -30 25 TestProxiedServerShowWatch -30 25 TestProxiedServerUpdateIfGuards -30 25 TestProxiedServerWispCreateRejectsEnumViolatingVar +44 25 TestProxiedServerHeartbeat +44 25 TestProxiedServerMolLastActivity +44 25 TestProxiedServerReassignFence +44 25 TestProxiedServerStatuses +44 25 TestProxiedServerUpdateWisp -30 26 TestProxiedServerFindDuplicates -30 26 TestProxiedServerGateCheck -30 26 TestProxiedServerGateFrontDoorDirectServerProxyParity -30 26 TestProxiedServerGatedRendering -30 26 TestProxiedServerServeAddComment -30 26 TestProxiedServerServeSettingsWrites -30 26 TestProxiedServerState +44 26 TestProxiedServerGraph +44 26 TestProxiedServerMolBond +44 26 TestProxiedServerServeReadRevision +44 26 TestProxiedServerTodo +44 26 TestProxiedServerUpdateHooks -30 27 TestProxiedServerBlocked -30 27 TestProxiedServerIfRevisionGuardClose -30 27 TestProxiedServerMolLastActivity -30 27 TestProxiedServerMolStale -30 27 TestProxiedServerMoleculeFrontDoorWriteParity -30 27 TestProxiedServerServeActivatesTheEventsJournal -30 27 TestProxiedServerServeRefusesAStaleCheckpointWithGone +44 27 TestProxiedServerCreate2B +44 27 TestProxiedServerGateList +44 27 TestProxiedServerGatedRendering +44 27 TestProxiedServerReopenConcurrent +44 27 TestProxiedServerSQLStatementRouting -30 28 TestProxiedServerComment -30 28 TestProxiedServerExport -30 28 TestProxiedServerList -30 28 TestProxiedServerReopen -30 28 TestProxiedServerServeCreate -30 28 TestProxiedServerUnclaim +44 28 TestProxiedServerComment +44 28 TestProxiedServerMolProgress +44 28 TestProxiedServerServeCount +44 28 TestProxiedServerServeRemoveDependency +44 28 TestProxiedServerUpdateB -30 29 TestProxiedServerDeferAutoWake -30 29 TestProxiedServerDep -30 29 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge -30 29 TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix -30 29 TestProxiedServerIfRevisionGuardDelete -30 29 TestProxiedServerTag -30 29 TestProxiedServerWispList +44 29 TestProxiedServerCreate2C +44 29 TestProxiedServerDep +44 29 TestProxiedServerDepConcurrent +44 29 TestProxiedServerServeSettingsWrites +44 29 TestProxiedServerWispCreateRejectsEnumViolatingVar -30 30 TestProxiedServerEventsJournalOffByDefault -30 30 TestProxiedServerMolSquash -30 30 TestProxiedServerPrune -30 30 TestProxiedServerPurgeWispsPlaneRetention -30 30 TestProxiedServerServeRelease -30 30 TestProxiedServerUpdateConcurrentClaim +44 30 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge +44 30 TestProxiedServerInfo +44 30 TestProxiedServerPrime +44 30 TestProxiedServerUpdateC +44 30 TestServerModeServeSkipsPostRunMaintenance + +44 31 TestProxiedServerCleanDatabasesPurgeDropped +44 31 TestProxiedServerGateLifecycle +44 31 TestProxiedServerPurgeWispsPlaneRetention +44 31 TestProxiedServerShow3B +44 31 TestServerModeDeferAutoWake + +44 32 TestProxiedServerClose3 +44 32 TestProxiedServerGC +44 32 TestProxiedServerList +44 32 TestProxiedServerReadyConcurrent +44 32 TestProxiedServerServeModeGate + +44 33 TestProxiedServerDeferUndefer +44 33 TestProxiedServerDepB +44 33 TestProxiedServerInitExplicitBeadsDirIsAuthoritative +44 33 TestProxiedServerMolBurn +44 33 TestProxiedServerOrphans + +44 34 TestProxiedServerChildren +44 34 TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix +44 34 TestProxiedServerPromote +44 34 TestProxiedServerServeRelease +44 34 TestProxiedServerUpdate2C + +44 35 TestProxiedServerConfigConcurrent +44 35 TestProxiedServerReady2 +44 35 TestProxiedServerShow + +44 36 TestProxiedServerDep2B +44 36 TestProxiedServerGateCreateInheritsRepoMetadata +44 36 TestProxiedServerServeRefusesTheJournalWhenItIsDisabled +44 36 TestProxiedServerWispGC + +44 37 TestProxiedServerDeleteWisp +44 37 TestProxiedServerNote +44 37 TestProxiedServerReady +44 37 TestProxiedServerShowWatch + +44 38 TestProxiedServerBackupRefusedOnExternalTopology +44 38 TestProxiedServerCascadeParity +44 38 TestProxiedServerClose4 +44 38 TestProxiedServerReadyClaimConcurrent +44 38 TestProxiedServerReclaim + +44 39 TestProxiedServerGateCheck +44 39 TestProxiedServerLabelB +44 39 TestProxiedServerMoleculeFrontDoorWriteParity + +44 40 TestProxiedServerDeleteConcurrent +44 40 TestProxiedServerIfRevisionGuardAssign +44 40 TestProxiedServerReadyC +44 40 TestProxiedServerRelate + +44 41 TestProxiedServerMolCurrent +44 41 TestProxiedServerPrune +44 41 TestProxiedServerTypes +44 41 TestProxiedServerUnclaim + +44 42 TestProxiedServerClose +44 42 TestProxiedServerCompactHistory +44 42 TestProxiedServerState +44 42 TestProxiedServerUpdateConcurrentMetadata + +44 43 TestProxiedServerCloseB +44 43 TestProxiedServerConfigB +44 43 TestProxiedServerMultiStatementSQL + +44 44 TestProxiedServerExport +44 44 TestProxiedServerGateCheckB +44 44 TestProxiedServerListSortRetiresTheWalk +44 44 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows +44 44 TestProxiedServerMolReadyGated +44 44 TestProxiedServerServeLifecycle diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index b3d13b0f6..5922cf041 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1732,8 +1732,8 @@ jobs: # Like pr-risk.yml's proxied-server tier, "Test (Proxied Dolt Cmd N/15)", # as --config=doltserver-proxied: //cmd/bd:bd_proxied_test runs the shard # script, so shard k runs exactly that manifest entry's tests. This lane - # runs 30 shards of its own (.github/scripts/proxied-cmd-test-shards.txt's - # duration-balanced 30-block; see scripts/ci/proxied_test_durations.json), + # runs 44 shards of its own (.github/scripts/proxied-cmd-test-shards.txt's + # duration-balanced 44-block; see scripts/ci/proxied_test_durations.json), # not the legacy jobs' 15 — shard k here is not the same split as their # shard k. Same build as --config=doltserver (race bd_test, non-race # bd_for_tests), shared with bazel-doltserver through the remote cache. @@ -1793,35 +1793,15 @@ jobs: fi # Remote: each shard's Go timeout is its job's 15m, 1200s Bazel cap, - # plus a compile. scripts/ci/proxied_test_durations.json's 176 entries - # are each a clean, single-job (no concurrent heavy bazel/dolt job - # intentionally started on the box; see that file's header for its - # ambient-contention caveat), -test.parallel=4 top-level-test - # duration — the exact config this job uses — from a single fresh, - # manifest-aligned 30-shard sweep's own (errors="0", - # so a failing-but-complete suite's time is still trustworthy); every - # one of the 176 discovered tests completed inside its shard run, so - # no `bazel run` leaf-summation workaround was needed this time. See - # that file's header for the full methodology, including why an - # earlier -test.parallel=1 (serial) capture was replaced: serial - # timings don't model this lane (which always runs parallel=4), and - # an independent re-review measured near-zero correlation (r=0.08) - # between the serial-derived predicted shard load and actual - # parallel=4 wall time. - # - # Measured per-shard wall time for the full 30-shard local sweep that - # produced this data: shards ranged from roughly 32s to 341s, with - # most clustering well under this job's 25m step timeout / 30m job - # timeout. After repacking with these corrected weights, the LPT - # estimate clusters every shard within a few seconds of ~355-360s - # (sum of all 176 measured durations / 30 shards) — no single test or - # small group of tests dominates a shard the way the discarded serial - # data wrongly implied. This is a local, single-job measurement, not - # a verified RBE forecast (this job has only ever run remote); if a - # shard's RBE wall time diverges sharply from this local estimate, - # that is RBE throughput differing from local execution, not - # necessarily a regression — compare against this job's own recent - # run history before assuming a hang. + # plus a compile. The 44-shard block is LPT-packed by + # scripts/ci/proxied_test_durations.json, a rbe-west capture of each + # top-level test's summed leaf-test time (see its header). Packed to + # <=430 cost-seconds per shard, and a shard's wall time ran ~0.3x its + # packed cost on the lane's executors (-test.parallel=4), so shards + # are expected around 100-150s. Nearly all of that is `bd init + # --proxied-server` against the shard's dolt sql-server, ~one per + # subtest; a shard running far longer than its packed cost is worth + # comparing against this job's recent history before assuming a hang. - name: bazel test //... --config=doltserver-proxied id: test timeout-minutes: 25 @@ -1850,7 +1830,7 @@ jobs: run: >- python3 tools/bazel/check_shard_coverage.py --bep "$RUNNER_TEMP/bazel-bep.json" - --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 30 + --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 44 - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index a38bc8bd7..b5e03704f 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1441,8 +1441,8 @@ sh_test( tags = ["embedded"], ) -# Runs 30 Bazel shards against the TestProxiedServer*/TestServerMode* -# functions, assigned by the manifest's 30-shard block (duration-balanced; +# Runs 44 Bazel shards against the TestProxiedServer*/TestServerMode* +# functions, assigned by the manifest's 44-shard block (duration-balanced; # see scripts/ci/proxied_test_durations.json), against this configuration's # (race) bd_test with BEADS_TEST_PROXIED_SERVER=1 and the non-race # bd_for_tests as the subprocess binary. This is a *different* shard count @@ -1499,7 +1499,7 @@ sh_test( # 192-core worker). "GOMAXPROCS": "4", }, - shard_count = 30, + shard_count = 44, tags = ["dolt-server-proxied"], ) diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index bbbcfba79..0e349f740 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -283,6 +283,14 @@ func TestProxiedServerClose(t *testing.T) { t.Errorf("close_reason: got %q, want %q", got, "wontfix") } }) +} + +// TestProxiedServerCloseB holds more of TestProxiedServerClose's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerCloseB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("close_with_comment_alias", func(t *testing.T) { t.Parallel() @@ -348,7 +356,6 @@ func TestProxiedServerClose(t *testing.T) { t.Errorf("expected 'not found' error, got: %s", out) } }) - } // TestProxiedServerClose3 is the second half of TestProxiedServerClose, @@ -754,6 +761,14 @@ func TestProxiedServerClose2(t *testing.T) { t.Errorf("expected single-issue error, got: %s", out) } }) +} + +// TestProxiedServerClose2B holds more of TestProxiedServerClose2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerClose2B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("close_suggest_next_multiple_ids_fails", func(t *testing.T) { t.Parallel() diff --git a/cmd/bd/config_proxied_integration_test.go b/cmd/bd/config_proxied_integration_test.go index 745142fc0..8cd1092f7 100644 --- a/cmd/bd/config_proxied_integration_test.go +++ b/cmd/bd/config_proxied_integration_test.go @@ -165,6 +165,14 @@ func TestProxiedServerConfig(t *testing.T) { t.Error("expected test.removeme to be absent from config list after unset") } }) +} + +// TestProxiedServerConfigB holds more of TestProxiedServerConfig's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerConfigB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("config_get_missing_key", func(t *testing.T) { t.Parallel() diff --git a/cmd/bd/create_proxied_integration_test.go b/cmd/bd/create_proxied_integration_test.go index 743c51e92..3ce839369 100644 --- a/cmd/bd/create_proxied_integration_test.go +++ b/cmd/bd/create_proxied_integration_test.go @@ -183,6 +183,14 @@ func TestProxiedServerCreate2(t *testing.T) { t.Errorf("metadata: got %v, want key=value", m) } }) +} + +// TestProxiedServerCreate2B holds more of TestProxiedServerCreate2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerCreate2B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("dry_run", func(t *testing.T) { t.Parallel() @@ -349,6 +357,14 @@ A new feature t.Errorf("source_repo: got %q, want %q", sourceRepo, "/path/to/repo") } }) +} + +// TestProxiedServerCreate2C holds more of TestProxiedServerCreate2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerCreate2C(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) // RULING R1 (TestParityCreateOnOccupiedIDRefuses): `bd create --id` on an // occupied ID refuses with exit 1 and a fixed message, leaving the diff --git a/cmd/bd/dep_proxied_integration_test.go b/cmd/bd/dep_proxied_integration_test.go index df0a41546..b8b9a315d 100644 --- a/cmd/bd/dep_proxied_integration_test.go +++ b/cmd/bd/dep_proxied_integration_test.go @@ -224,7 +224,18 @@ func TestProxiedServerDep(t *testing.T) { t.Errorf("expected 'cycle' error: %s", out) } }) + }) +} +// TestProxiedServerDepB holds more of TestProxiedServerDep's Add cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerDepB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + t.Run("Add", func(t *testing.T) { + t.Parallel() t.Run("child_parent_antipattern", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "dpa8") @@ -379,7 +390,6 @@ func TestProxiedServerDep(t *testing.T) { } }) }) - } func TestProxiedServerDep2(t *testing.T) { @@ -550,6 +560,15 @@ func TestProxiedServerDep2(t *testing.T) { } }) }) +} + +// TestProxiedServerDep2B is the next slice of TestProxiedServerDep2's cases, a +// separate top-level test so the bd_proxied_test shard manifest can place it +// on its own shard. +func TestProxiedServerDep2B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("Tree", func(t *testing.T) { t.Parallel() @@ -635,7 +654,18 @@ func TestProxiedServerDep2(t *testing.T) { bdProxiedDep(t, bd, p.dir, "add", root.ID, dep.ID) _ = bdProxiedDep(t, bd, p.dir, "tree", root.ID, "--status", "open") }) + }) +} +// TestProxiedServerDep2C holds more of TestProxiedServerDep2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerDep2C(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + t.Run("Tree", func(t *testing.T) { + t.Parallel() t.Run("format_mermaid", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "dpt7") diff --git a/cmd/bd/gate_check_proxied_integration_test.go b/cmd/bd/gate_check_proxied_integration_test.go index 633b854d5..bdc1051b1 100644 --- a/cmd/bd/gate_check_proxied_integration_test.go +++ b/cmd/bd/gate_check_proxied_integration_test.go @@ -126,6 +126,17 @@ func TestProxiedServerGateCheck(t *testing.T) { t.Errorf("expected 1 resolved after target close, got:\n%s", out) } }) +} + +// TestProxiedServerGateCheckB holds more of TestProxiedServerGateCheck's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerGateCheckB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + const expiredTimeout = time.Hour + longAgo := func() time.Time { return time.Now().UTC().Add(-48 * time.Hour) } t.Run("bead_gate_resolves_when_seen_target_is_deleted", func(t *testing.T) { t.Parallel() diff --git a/cmd/bd/label_proxied_integration_test.go b/cmd/bd/label_proxied_integration_test.go index be84341cc..7cfd1d94d 100644 --- a/cmd/bd/label_proxied_integration_test.go +++ b/cmd/bd/label_proxied_integration_test.go @@ -230,6 +230,15 @@ func TestProxiedServerLabel(t *testing.T) { } } }) +} + +// TestProxiedServerLabelB holds more of TestProxiedServerLabel's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerLabelB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("propagate_no_children", func(t *testing.T) { t.Parallel() @@ -382,6 +391,15 @@ func TestProxiedServerLabel(t *testing.T) { t.Errorf("expected both labels in text list, got:\n%s", out) } }) +} + +// TestProxiedServerLabelC holds more of TestProxiedServerLabel's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerLabelC(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("propagate_json", func(t *testing.T) { t.Parallel() diff --git a/cmd/bd/ready_proxied_integration_test.go b/cmd/bd/ready_proxied_integration_test.go index 758a62a64..69d83fcb7 100644 --- a/cmd/bd/ready_proxied_integration_test.go +++ b/cmd/bd/ready_proxied_integration_test.go @@ -238,6 +238,14 @@ func TestProxiedServerReady(t *testing.T) { envelope.Pagination.Total, s[start:]) } }) +} + +// TestProxiedServerReadyB holds more of TestProxiedServerReady's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerReadyB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("pagination_envelope_not_truncated", func(t *testing.T) { t.Parallel() @@ -409,6 +417,14 @@ func TestProxiedServerReady(t *testing.T) { t.Error("after re-fetch: Assignee empty (commit may not have persisted)") } }) +} + +// TestProxiedServerReadyC holds more of TestProxiedServerReady's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerReadyC(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("blocked_issues_excluded_from_default_output", func(t *testing.T) { t.Parallel() @@ -536,7 +552,6 @@ func TestProxiedServerReady(t *testing.T) { t.Errorf("expected 'No molecules ready for gate-resume dispatch' text, got: %s", text) } }) - } func TestProxiedServerReady2(t *testing.T) { diff --git a/cmd/bd/reopen_proxied_integration_test.go b/cmd/bd/reopen_proxied_integration_test.go index 3d5884f11..68e667e73 100644 --- a/cmd/bd/reopen_proxied_integration_test.go +++ b/cmd/bd/reopen_proxied_integration_test.go @@ -291,6 +291,14 @@ func TestProxiedServerReopen(t *testing.T) { t.Errorf("status: got %q, want open", got) } }) +} + +// TestProxiedServerReopenB holds more of TestProxiedServerReopen's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerReopenB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) // The other side of the category rule, and the same line the direct route // prints: a status that is neither done nor open is left alone and said so. diff --git a/cmd/bd/show_proxied_integration_test.go b/cmd/bd/show_proxied_integration_test.go index 32a914ce8..b518aac7d 100644 --- a/cmd/bd/show_proxied_integration_test.go +++ b/cmd/bd/show_proxied_integration_test.go @@ -218,6 +218,14 @@ func TestProxiedServerShow(t *testing.T) { t.Errorf("expected comments with --include-comments: %v", m) } }) +} + +// TestProxiedServerShowB holds more of TestProxiedServerShow's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerShowB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) // ga-clgh: proxied path mirror of the direct-path fix (a2a69f5e9). The // proxied handler used to leave comments_omitted unset entirely, so a @@ -398,7 +406,6 @@ func TestProxiedServerShow(t *testing.T) { t.Errorf("expected title in output: %s", out) } }) - } func TestProxiedServerShow2(t *testing.T) { @@ -591,6 +598,14 @@ func TestProxiedServerShow2(t *testing.T) { t.Errorf("expected --refs output to contain referrer %s: %s", child.ID, out) } }) +} + +// TestProxiedServerShow2B holds more of TestProxiedServerShow2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerShow2B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("show_refs_groups_by_type", func(t *testing.T) { t.Parallel() @@ -754,7 +769,6 @@ func TestProxiedServerShow2(t *testing.T) { t.Errorf("expected wisp child %s in --children: %s", childW.ID, out) } }) - } func TestProxiedServerShow3(t *testing.T) { @@ -887,6 +901,14 @@ func TestProxiedServerShow3(t *testing.T) { arr[0]["id"], arr[1]["id"], root.ID, reply.ID) } }) +} + +// TestProxiedServerShow3B holds more of TestProxiedServerShow3's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerShow3B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("show_thread_orphan_message_renders_alone", func(t *testing.T) { t.Parallel() diff --git a/cmd/bd/update_proxied_integration_test.go b/cmd/bd/update_proxied_integration_test.go index efea4da05..cb8c44735 100644 --- a/cmd/bd/update_proxied_integration_test.go +++ b/cmd/bd/update_proxied_integration_test.go @@ -213,6 +213,14 @@ func TestProxiedServerUpdate(t *testing.T) { t.Errorf("lost issue %s assignee = %q, want alice (unchanged)", lost.ID, gotLost.Assignee) } }) +} + +// TestProxiedServerUpdateB holds more of TestProxiedServerUpdate's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerUpdateB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) // Parity with the non-proxied TestMultiIDUpdatePartialFailureExitsNonzero: // a generic per-ID failure (a bogus ID) between two good IDs must exit @@ -427,6 +435,14 @@ func TestProxiedServerUpdate(t *testing.T) { t.Errorf("dependent should be unblocked after blocker closes") } }) +} + +// TestProxiedServerUpdateC holds more of TestProxiedServerUpdate's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerUpdateC(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("invalid_status_rejected", func(t *testing.T) { t.Parallel() @@ -560,7 +576,6 @@ func TestProxiedServerUpdate(t *testing.T) { before, after) } }) - } func TestProxiedServerUpdate2(t *testing.T) { @@ -685,6 +700,14 @@ func TestProxiedServerUpdate2(t *testing.T) { t.Errorf("ephemeral: got %d, want 0 after --persistent", ephemeral) } }) +} + +// TestProxiedServerUpdate2B holds more of TestProxiedServerUpdate2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerUpdate2B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("update_ephemeral", func(t *testing.T) { t.Parallel() @@ -809,6 +832,14 @@ func TestProxiedServerUpdate2(t *testing.T) { t.Errorf("metadata[keep]: got %v, want %q", got["keep"], "yes") } }) +} + +// TestProxiedServerUpdate2C holds more of TestProxiedServerUpdate2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerUpdate2C(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("update_set_metadata", func(t *testing.T) { t.Parallel() @@ -933,7 +964,6 @@ func TestProxiedServerUpdate2(t *testing.T) { t.Errorf("defer_until: got nil, want non-nil (defer still applied)") } }) - } func TestProxiedServerUpdate3(t *testing.T) { diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 15ba5624c..93ec7f4cd 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1041,7 +1041,7 @@ Required` requires them to have run remotely and passed. frozen 20-shard block was the retired `test-embedded-cmd` jobs' split, F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; the frozen 5-shard block was `test-embedded-storage`'s, F1), - `//cmd/bd:bd_proxied_test` (30; the frozen 15-shard block was + `//cmd/bd:bd_proxied_test` (44; the frozen 15-shard block was `test-proxied-cmd`'s, F2) and `//internal/storage/dolt:dolt_server_full_test` (16) to have run exactly the tests its shard script lists (list-only mode, minus diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 41efd4955..2f293a23b 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -42,7 +42,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | | Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | -| Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 30 shards, each with a Dolt server. | +| Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 44 shards, each with a Dolt server. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | @@ -184,7 +184,7 @@ builds with the integration tag like `--config=integration`. Each shard runs its CI job's shard script, so for `--config=doltserver-integration` Bazel shard k runs the tests of job k+1 (both split the manifest's 16-shard block the same way). `--config=doltserver-proxied`'s `bd_proxied_test` instead -runs the manifest's own 30-shard block — bin-packed by measured duration, +runs the manifest's own 44-shard block — bin-packed by measured duration, not the legacy jobs' 15-shard, bd-init-cost-proxy block — so shard k there is not job k+1's tests; it is a different split of the same tests. diff --git a/scripts/ci/gen_proxied_shard_manifest.py b/scripts/ci/gen_proxied_shard_manifest.py index 8ebea6c00..0a099e0f9 100644 --- a/scripts/ci/gen_proxied_shard_manifest.py +++ b/scripts/ci/gen_proxied_shard_manifest.py @@ -56,7 +56,7 @@ name coverage. The manifest file holds more than one block (one per total_shards value in use): the legacy, frozen 15-shard block pr-risk.yml/main.yml's jobs read, and -the Bazel-only lane's own block (30 shards as of F2). By default this prints +the Bazel-only lane's own block (44 shards since 2026-10-09). By default this prints only the requested total's block to stdout, which is NOT safe to redirect straight into the manifest file (`gen... 30 > file` deletes every other block). Use --write to update the file in place instead: it replaces only diff --git a/scripts/ci/proxied_test_durations.json b/scripts/ci/proxied_test_durations.json index 8e1e5f3f1..27297aae8 100644 --- a/scripts/ci/proxied_test_durations.json +++ b/scripts/ci/proxied_test_durations.json @@ -1,250 +1,254 @@ { "_comment": [ - "Measured locally with `bazel --nohome_rc test --config=doltserver-proxied", - "//cmd/bd:bd_proxied_test` -- the exact config bazel.yml's proxied-server tier", - "uses, including `-test.parallel=4` (local sandboxed execution only; no remote", - "executor or cache flags were configured for this measurement run; Bazel's own", - "--remote_download_regex etc. in that config do not enable remote execution", - "without --config=remote-exec, which was not passed).", + "Measured on rbe-west, the lane's own remote executors (instance oss), on", + "2026-10-09: two `bazel test //cmd/bd:bd_proxied_test", + "--config=doltserver-proxied --nocache_test_results` runs, one against the", + "then-committed 30-shard block and one against a 40-shard repack, so every", + "discovered test ran exactly once per run, with the lane's -test.parallel=4", + "and GOMAXPROCS=4. Both passed.", "", - "This replaces an earlier capture that used `-test.parallel=1` (serial). That", - "methodology avoided double-counting a parallel subtest group's own duration on", - "top of its children's, which was a genuine bug in an even earlier pass -- but", - "serial timings do not model this lane, which always runs at parallel=4, and an", - "independent re-review measured near-zero correlation (r=0.08) between the", - "serial-derived predicted shard load and actual parallel=4 wall time. These", - "values are each a clean, single-job (no other concurrent heavy bazel/dolt job", - "intentionally started on the box; see below), parallel=4 top-level test's own", - " attribute from a single fresh, manifest-aligned 30-shard", - "sweep where every shard ran its exact assigned test set once (no duplicates,", - "no gaps -- see scripts/ci/gen_proxied_shard_manifest.py's exact-once coverage", - "check). A top-level Go test's already reflects, not adds to,", - "its own parallel children's wall-clock time, so no leaf-summation step was", - "needed this time: every one of the 176 discovered tests completed inside a", - "single shard run with errors=\"0\" (none needed Bazel's per-target default", - "timeout worked around via a separate `bazel run`, unlike the prior capture).", + "Each value is a top-level test's serial-equivalent cost: the sum of the", + " of every leaf test under it (a subtest with no subtests of", + "its own; the test itself when it has none), each leaf averaged over the", + "two runs. It is NOT , which the previous (local) capture", + "used: for a t.Parallel test that attribute runs from the test's first RUN", + "line to its last PASS line, so it also counts the time the test sat paused", + "behind its shard's other tests. That made a cheap test that happened to", + "wait look as heavy as the shard, which is why the previous 30-shard block,", + "packed to ~360s per shard on paper, ran 38-286s per shard in CI. Leaves,", + "not direct subtests, because a parallel group's own is 0.", "", - "`seconds_per_init_fallback` is the weighted average seconds-per-init over", - "every measured test in this file (sum of all durations / sum of all inits),", - "used only for a brand-new test with no entry yet (see", - "scripts/ci/gen_proxied_shard_manifest.py's discover_inits_cost()). It dropped", - "from the serial capture's 39.7275 to this capture's lower value because", - "parallel=4 genuinely shortens most tests' wall time relative to inits count;", - "this is expected, not a data error.", + "A shard's remote wall time came out at about 0.3x its packed cost (four", + "subtests run at once): correlation 0.90 over the 30-shard run (19,224", + "cost-seconds, 5,671 shard-seconds of wall time); the 40-shard run's", + "shards ran 71-162s, apart from one that held two then-unsplit tests whose", + "cost the first capture had missed.", "", - "This file still does not claim shard totals land within a few seconds of each", - "other -- see .github/workflows/bazel.yml for the current, honest expected", - "spread for the 30-shard proxied lane, now also derived from this same sweep.", + "The cost is almost all `bd init --proxied-server` against the shared dolt", + "sql-server: each init issues ~2,000 statements (81 DOLT_COMMITs) and costs", + "the server ~6-12 CPU-seconds, and most subtests run one init. A top-level", + "test with ~20 single-init subtests therefore cost 600-920 and ran for up", + "to ~350s alone, which no shard count can pack under that. The fifteen", + "heaviest were split into two or three top-level functions", + "(TestProxiedServerUpdate, -UpdateB, -UpdateC, ...); a split function's", + "entry is the sum of the leaves it now holds.", "", - "The box this was measured on also runs other engineers'/agents' unrelated", - "Bazel jobs; this capture was not run with exclusive use of the machine, only", - "with no other *intentional, task-related* heavy bazel/dolt job started", - "concurrently by this task. Absolute seconds may be inflated by ambient", - "contention; relative ordering between tests, which is what LPT packing", - "actually depends on, should still be far more representative than the", - "discarded serial data (confirmed: 30/30 shards passed, 0 failures, 0 races,", - "with the two blocker fixes -- shortSocketPath and cmd.Cancel -- applied).", + "`seconds_per_init_fallback` is the sum of all costs over the sum of all", + "inits (discover_inits_cost()), used only for a test with no entry yet.", "", - "Regenerate with a local `bazel --nohome_rc test --config=doltserver-proxied", - "//cmd/bd:bd_proxied_test --build_event_json_file=` sweep, then pull", - "each shard's bazel-testlogs test.xml per top-level test --", - "not from downloaded CI artifacts -- so the parallel=4, CI-representative", - "methodology above is applied consistently.", - "", - "Added after that sweep: TestProxiedServerOutageReconnectAcceptanceMatrix (80.0,", - "renamed into the discovery pattern once its outage bound was fixed). Not a", - "single sweep's value: local --config=doltserver-proxied runs measured it at", - "55-79 s alone, 60-70 s inside its shard, and a reviewer's 68-109 s under load;", - "80 s is a rounded middle of that spread. seconds_per_init_fallback was", - "recomputed to include it.", - "", - "TestProxiedServerClose / Close3 / Close4 (gastownhall/beads#7151): the 174.6s", - "capture above was taken for the undivided TestProxiedServerClose (23 bd-init", - "subtests). The suite was then split after close_nonexistent_id and again after", - "close_pinned_with_force, so the three entries hold that one capture", - "apportioned by bd-init count (11, 6 and 6 of 23), not fresh measurements; the", - "next sweep replaces all three." + "Regenerate the same way: fresh remote runs of the target, then sum each", + "top-level test's leaf s from every shard's", + "bazel-testlogs/cmd/bd/bd_proxied_test/shard_*/test.xml." ], - "seconds_per_init_fallback": 17.2952, + "seconds_per_init_fallback": 29.5374, "tests": { - "TestProxiedServerAssign": 38.67, - "TestProxiedServerAutoPrune": 67.5, - "TestProxiedServerBackupRefusedOnExternalTopology": 5.96, - "TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce": 9.18, - "TestProxiedServerBatchParity": 60.04, - "TestProxiedServerBlocked": 138.63, - "TestProxiedServerCascadeParity": 78.23, - "TestProxiedServerChildren": 11.28, - "TestProxiedServerCleanDatabases": 7.14, - "TestProxiedServerCleanDatabasesPurgeDropped": 7.76, - "TestProxiedServerClose": 83.5, - "TestProxiedServerClose2": 218.83, - "TestProxiedServerClose3": 45.55, - "TestProxiedServerClose4": 45.55, - "TestProxiedServerCloseConcurrent": 38.25, - "TestProxiedServerCloseSiblingBlockersFanIn": 99.71, - "TestProxiedServerComment": 64.04, - "TestProxiedServerCompactDolt": 11.55, - "TestProxiedServerCompactHistory": 12.38, - "TestProxiedServerConfig": 188.91, - "TestProxiedServerConfigConcurrent": 48.59, - "TestProxiedServerConfigSetMany": 196.57, - "TestProxiedServerContext": 72.27, - "TestProxiedServerCount": 60.65, - "TestProxiedServerCountIncludeInfra": 34.69, - "TestProxiedServerCreate": 46.42, - "TestProxiedServerCreate2": 165.07, - "TestProxiedServerCreateInfraTypeRoutesToWisps": 142.17, - "TestProxiedServerDeferAutoWake": 12.95, - "TestProxiedServerDeferUndefer": 51.14, - "TestProxiedServerDelete": 44.82, - "TestProxiedServerDeleteBlockedJSONSingleDoc": 8.09, - "TestProxiedServerDeleteConcurrent": 10.57, - "TestProxiedServerDeleteForceOrphans": 9.16, - "TestProxiedServerDeleteWisp": 7.06, - "TestProxiedServerDep": 121.62, - "TestProxiedServerDep2": 227.28, - "TestProxiedServerDepConcurrent": 30.55, - "TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge": 10.13, - "TestProxiedServerDoltLifecycle": 6.18, - "TestProxiedServerDoltRemoteRemove": 72.67, - "TestProxiedServerDuplicates": 44.77, - "TestProxiedServerEdit": 10.16, - "TestProxiedServerEpic": 109.49, - "TestProxiedServerEventsJournal": 6.48, - "TestProxiedServerEventsJournalOffByDefault": 65.83, - "TestProxiedServerExport": 26.44, - "TestProxiedServerExternalCreate": 5.73, - "TestProxiedServerExternalMutationPolicy": 25.0, - "TestProxiedServerExternalPolicyDirectParity": 18.12, - "TestProxiedServerFindDuplicates": 51.66, - "TestProxiedServerGC": 12.73, - "TestProxiedServerGateCheck": 140.99, - "TestProxiedServerGateCreateInheritsRepoMetadata": 47.1, - "TestProxiedServerGateFrontDoorDirectServerProxyParity": 17.36, - "TestProxiedServerGateLifecycle": 49.79, - "TestProxiedServerGateList": 41.19, - "TestProxiedServerGraph": 15.43, - "TestProxiedServerHeartbeat": 57.23, - "TestProxiedServerHistory": 21.81, - "TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix": 59.61, - "TestProxiedServerHistoryRemoteSupportedFrontDoorParity": 5.97, - "TestProxiedServerHuman": 263.77, - "TestProxiedServerImport": 160.85, - "TestProxiedServerInfo": 28.32, - "TestProxiedServerInitExplicitBeadsDirIsAuthoritative": 11.99, - "TestProxiedServerInitIfMissing": 6.69, - "TestProxiedServerKV": 94.24, - "TestProxiedServerLabel": 179.25, - "TestProxiedServerLink": 51.49, - "TestProxiedServerLint": 15.51, - "TestProxiedServerList": 112.68, - "TestProxiedServerListComments": 31.13, - "TestProxiedServerListSortRetiresTheWalk": 46.29, - "TestProxiedServerListWatchCountsTheOffsetIntoMaxRows": 10.54, - "TestProxiedServerMemory": 58.87, - "TestProxiedServerMolBond": 79.34, - "TestProxiedServerMolBurn": 253.65, - "TestProxiedServerMolCurrent": 85.1, - "TestProxiedServerMolDistill": 78.03, - "TestProxiedServerMolLastActivity": 111.33, - "TestProxiedServerMolProgress": 182.51, - "TestProxiedServerMolReadyGated": 66.34, - "TestProxiedServerMolShow": 36.15, - "TestProxiedServerMolSquash": 118.23, - "TestProxiedServerMolStale": 53.95, - "TestProxiedServerMoleculeFrontDoorDirectServerProxyParity": 22.98, - "TestProxiedServerMoleculeFrontDoorWriteParity": 19.52, - "TestProxiedServerMultiStatementSQL": 6.76, - "TestProxiedServerNote": 11.21, - "TestProxiedServerOrphans": 12.11, - "TestProxiedServerOutageReconnectAcceptanceMatrix": 80.0, + "TestProxiedServerAssign": 55.49, + "TestProxiedServerAutoPrune": 6.71, + "TestProxiedServerBackupRefusedOnExternalTopology": 0.45, + "TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce": 10.13, + "TestProxiedServerBatchParity": 64.1, + "TestProxiedServerBlocked": 49.25, + "TestProxiedServerCascadeParity": 167.07, + "TestProxiedServerChildren": 12.43, + "TestProxiedServerCleanDatabases": 1.02, + "TestProxiedServerCleanDatabasesPurgeDropped": 2.23, + "TestProxiedServerClose": 223.82, + "TestProxiedServerClose2": 379.25, + "TestProxiedServerClose2B": 372.47, + "TestProxiedServerClose3": 253.62, + "TestProxiedServerClose4": 236.01, + "TestProxiedServerCloseB": 199.51, + "TestProxiedServerCloseBeadGate": 85.6, + "TestProxiedServerCloseConcurrent": 39.05, + "TestProxiedServerCloseSiblingBlockersFanIn": 45.72, + "TestProxiedServerComment": 101.7, + "TestProxiedServerCompactDolt": 0.15, + "TestProxiedServerCompactHistory": 2.27, + "TestProxiedServerConfig": 281.3, + "TestProxiedServerConfigB": 221.92, + "TestProxiedServerConfigConcurrent": 21.07, + "TestProxiedServerConfigSetMany": 56.32, + "TestProxiedServerContext": 30.68, + "TestProxiedServerCount": 29.54, + "TestProxiedServerCountIncludeInfra": 7.0, + "TestProxiedServerCreate": 35.39, + "TestProxiedServerCreate2": 313.1, + "TestProxiedServerCreate2B": 266.11, + "TestProxiedServerCreate2C": 115.17, + "TestProxiedServerCreateInfraTypeRoutesToWisps": 54.19, + "TestProxiedServerDeferAutoWake": 10.55, + "TestProxiedServerDeferUndefer": 23.62, + "TestProxiedServerDelete": 30.3, + "TestProxiedServerDeleteBlockedJSONSingleDoc": 16.89, + "TestProxiedServerDeleteConcurrent": 19.41, + "TestProxiedServerDeleteForceOrphans": 34.51, + "TestProxiedServerDeleteWisp": 4.98, + "TestProxiedServerDep": 265.67, + "TestProxiedServerDep2": 421.17, + "TestProxiedServerDep2B": 245.92, + "TestProxiedServerDep2C": 287.95, + "TestProxiedServerDepB": 252.49, + "TestProxiedServerDepConcurrent": 29.34, + "TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge": 23.65, + "TestProxiedServerDoltLifecycle": 0.19, + "TestProxiedServerDoltRemoteRemove": 79.1, + "TestProxiedServerDuplicates": 76.36, + "TestProxiedServerEdit": 19.49, + "TestProxiedServerEpic": 47.73, + "TestProxiedServerEventsJournal": 6.01, + "TestProxiedServerEventsJournalOffByDefault": 5.45, + "TestProxiedServerExport": 18.56, + "TestProxiedServerExternalCreate": 5.6, + "TestProxiedServerExternalMutationPolicy": 15.57, + "TestProxiedServerExternalPolicyDirectParity": 8.37, + "TestProxiedServerFindDuplicates": 51.33, + "TestProxiedServerGC": 5.82, + "TestProxiedServerGateCheck": 171.29, + "TestProxiedServerGateCheckB": 205.0, + "TestProxiedServerGateCreateInheritsRepoMetadata": 4.94, + "TestProxiedServerGateFrontDoorDirectServerProxyParity": 22.23, + "TestProxiedServerGateLifecycle": 124.1, + "TestProxiedServerGateList": 48.97, + "TestProxiedServerGatedRendering": 9.09, + "TestProxiedServerGraph": 8.3, + "TestProxiedServerHeartbeat": 11.7, + "TestProxiedServerHistory": 7.88, + "TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix": 26.2, + "TestProxiedServerHistoryRemoteSupportedFrontDoorParity": 5.58, + "TestProxiedServerHuman": 358.37, + "TestProxiedServerIfRevisionGuardAssign": 0.66, + "TestProxiedServerIfRevisionGuardClose": 0.82, + "TestProxiedServerIfRevisionGuardDelete": 0.81, + "TestProxiedServerIfRevisionGuardReopen": 1.48, + "TestProxiedServerIfRevisionGuardUpdate": 0.65, + "TestProxiedServerImport": 288.64, + "TestProxiedServerInfo": 6.0, + "TestProxiedServerInitExplicitBeadsDirIsAuthoritative": 12.59, + "TestProxiedServerInitIfMissing": 0.45, + "TestProxiedServerKV": 340.45, + "TestProxiedServerLabel": 298.09, + "TestProxiedServerLabelB": 234.22, + "TestProxiedServerLabelC": 271.95, + "TestProxiedServerLink": 62.35, + "TestProxiedServerLint": 6.67, + "TestProxiedServerList": 131.93, + "TestProxiedServerListComments": 1.97, + "TestProxiedServerListSortRetiresTheWalk": 0.23, + "TestProxiedServerListWatchCountsTheOffsetIntoMaxRows": 0.83, + "TestProxiedServerMemory": 90.84, + "TestProxiedServerMolBond": 267.25, + "TestProxiedServerMolBurn": 135.47, + "TestProxiedServerMolCurrent": 177.89, + "TestProxiedServerMolDistill": 55.5, + "TestProxiedServerMolLastActivity": 52.81, + "TestProxiedServerMolProgress": 41.09, + "TestProxiedServerMolReadyGated": 202.08, + "TestProxiedServerMolShow": 83.77, + "TestProxiedServerMolSquash": 73.31, + "TestProxiedServerMolStale": 52.8, + "TestProxiedServerMoleculeFrontDoorDirectServerProxyParity": 2.14, + "TestProxiedServerMoleculeFrontDoorWriteParity": 21.32, + "TestProxiedServerMultiStatementSQL": 7.5, + "TestProxiedServerNote": 21.92, + "TestProxiedServerOrphans": 4.25, + "TestProxiedServerOutageReconnectAcceptanceMatrix": 36.22, "TestProxiedServerPathHelpers": 0.0, - "TestProxiedServerPing": 11.73, - "TestProxiedServerPour": 253.65, - "TestProxiedServerPourRejectsEnumViolatingVar": 21.66, - "TestProxiedServerPrime": 94.24, - "TestProxiedServerPriority": 41.35, - "TestProxiedServerPromote": 255.67, - "TestProxiedServerPrune": 119.06, - "TestProxiedServerPurgeWispsPlaneRetention": 24.55, - "TestProxiedServerQuery": 42.94, - "TestProxiedServerQuick": 22.7, - "TestProxiedServerReady": 238.66, - "TestProxiedServerReady2": 71.96, - "TestProxiedServerReadyClaimConcurrent": 20.86, - "TestProxiedServerReadyConcurrent": 11.34, - "TestProxiedServerReadyWisp": 255.67, - "TestProxiedServerReassignFence": 57.62, - "TestProxiedServerReclaim": 11.62, - "TestProxiedServerRecomputeBlocked": 19.71, - "TestProxiedServerRelate": 160.07, - "TestProxiedServerReopen": 125.1, - "TestProxiedServerReopenConcurrent": 66.34, - "TestProxiedServerSQL": 13.91, - "TestProxiedServerSQLDatabaseFlag": 12.28, - "TestProxiedServerSQLStatementRouting": 18.36, - "TestProxiedServerSearch": 78.35, - "TestProxiedServerServeActivatesTheEventsJournal": 9.87, - "TestProxiedServerServeAddComment": 25.29, - "TestProxiedServerServeAddDependencies": 15.59, - "TestProxiedServerServeBatchClose": 16.3, - "TestProxiedServerServeClaim": 16.32, - "TestProxiedServerServeClaimNext": 17.04, - "TestProxiedServerServeClose": 38.42, - "TestProxiedServerServeCount": 21.49, - "TestProxiedServerServeCreate": 11.84, - "TestProxiedServerServeDelete": 13.83, - "TestProxiedServerServeEdgeCounts": 20.14, - "TestProxiedServerServeLifecycle": 21.66, - "TestProxiedServerServeModeGate": 31.59, - "TestProxiedServerServeReadParity": 15.44, - "TestProxiedServerServeReadRevision": 10.4, - "TestProxiedServerServeReadsTheJournalItJustWrote": 38.39, - "TestProxiedServerServeRefusesAStaleCheckpointWithGone": 22.47, - "TestProxiedServerServeRefusesAStreamOnADisabledJournal": 42.35, - "TestProxiedServerServeRefusesTheJournalWhenItIsDisabled": 19.4, - "TestProxiedServerServeRelatedIssues": 28.93, - "TestProxiedServerServeRelease": 11.29, - "TestProxiedServerServeRemoveDependency": 12.52, - "TestProxiedServerServeSettingsWrites": 10.68, - "TestProxiedServerServeStreamsTheJournalItIsWriting": 11.89, - "TestProxiedServerServeUpdate": 17.56, - "TestProxiedServerServeUpdateClaim": 13.57, - "TestProxiedServerSetState": 15.28, - "TestProxiedServerShow": 216.16, - "TestProxiedServerShow2": 232.1, - "TestProxiedServerShow3": 157.69, - "TestProxiedServerShowWatch": 109.69, - "TestProxiedServerStale": 15.61, - "TestProxiedServerState": 110.26, - "TestProxiedServerStatus": 28.61, - "TestProxiedServerStatuses": 27.0, - "TestProxiedServerStorageClass": 150.78, - "TestProxiedServerSwarm": 29.89, - "TestProxiedServerTag": 33.46, - "TestProxiedServerTodo": 43.18, - "TestProxiedServerTypes": 12.51, - "TestProxiedServerTypesAgreeWithCreate": 14.34, - "TestProxiedServerUnclaim": 16.57, - "TestProxiedServerUnclaimIfAssigneeParity": 25.33, - "TestProxiedServerUpdate": 238.19, - "TestProxiedServerUpdate2": 170.45, - "TestProxiedServerUpdate3": 184.1, - "TestProxiedServerUpdateClosePolicy": 37.4, - "TestProxiedServerUpdateConcurrentAppendNotes": 6.6, - "TestProxiedServerUpdateConcurrentClaim": 17.68, - "TestProxiedServerUpdateConcurrentMetadata": 8.68, - "TestProxiedServerUpdateHooks": 20.14, - "TestProxiedServerUpdateIfGuards": 141.97, - "TestProxiedServerUpdateWisp": 82.23, - "TestProxiedServerWispCreate": 74.62, - "TestProxiedServerWispCreateRejectsEnumViolatingVar": 25.49, - "TestProxiedServerWispGC": 52.72, - "TestProxiedServerWispList": 118.23, - "TestServerModeDeferAutoWake": 32.77, - "TestServerModeServe": 19.28, - "TestServerModeServeSkipsPostRunMaintenance": 10.52, - "TestServerModeTypesAgreeWithCreate": 16.0 + "TestProxiedServerPing": 0.76, + "TestProxiedServerPour": 90.63, + "TestProxiedServerPourRejectsEnumViolatingVar": 16.32, + "TestProxiedServerPrime": 120.62, + "TestProxiedServerPriority": 29.69, + "TestProxiedServerPromote": 137.81, + "TestProxiedServerPrune": 229.41, + "TestProxiedServerPurgeWispsPlaneRetention": 27.89, + "TestProxiedServerQuery": 32.83, + "TestProxiedServerQuick": 32.66, + "TestProxiedServerReady": 237.31, + "TestProxiedServerReady2": 159.52, + "TestProxiedServerReadyB": 279.25, + "TestProxiedServerReadyC": 232.74, + "TestProxiedServerReadyClaimConcurrent": 21.86, + "TestProxiedServerReadyConcurrent": 14.36, + "TestProxiedServerReadyWisp": 315.55, + "TestProxiedServerReassignFence": 92.12, + "TestProxiedServerReclaim": 1.41, + "TestProxiedServerRecomputeBlocked": 69.58, + "TestProxiedServerRelate": 174.16, + "TestProxiedServerReopen": 317.88, + "TestProxiedServerReopenB": 283.24, + "TestProxiedServerReopenConcurrent": 100.49, + "TestProxiedServerSQL": 4.56, + "TestProxiedServerSQLDatabaseFlag": 14.98, + "TestProxiedServerSQLStatementRouting": 2.61, + "TestProxiedServerSearch": 79.86, + "TestProxiedServerServeActivatesTheEventsJournal": 10.7, + "TestProxiedServerServeAddComment": 3.85, + "TestProxiedServerServeAddDependencies": 7.7, + "TestProxiedServerServeBatchClose": 14.42, + "TestProxiedServerServeClaim": 13.9, + "TestProxiedServerServeClaimNext": 12.27, + "TestProxiedServerServeClose": 21.8, + "TestProxiedServerServeCount": 15.2, + "TestProxiedServerServeCreate": 2.62, + "TestProxiedServerServeDelete": 5.64, + "TestProxiedServerServeEdgeCounts": 15.27, + "TestProxiedServerServeLifecycle": 0.03, + "TestProxiedServerServeModeGate": 23.75, + "TestProxiedServerServeReadParity": 6.26, + "TestProxiedServerServeReadRevision": 3.59, + "TestProxiedServerServeReadsTheJournalItJustWrote": 14.23, + "TestProxiedServerServeRefusesAStaleCheckpointWithGone": 14.62, + "TestProxiedServerServeRefusesAStreamOnADisabledJournal": 14.96, + "TestProxiedServerServeRefusesTheJournalWhenItIsDisabled": 13.62, + "TestProxiedServerServeRelatedIssues": 14.9, + "TestProxiedServerServeRelease": 4.5, + "TestProxiedServerServeRemoveDependency": 5.12, + "TestProxiedServerServeSettingsWrites": 5.08, + "TestProxiedServerServeStreamsTheJournalItIsWriting": 21.99, + "TestProxiedServerServeUpdate": 14.38, + "TestProxiedServerServeUpdateClaim": 4.97, + "TestProxiedServerSetState": 14.99, + "TestProxiedServerShow": 246.84, + "TestProxiedServerShow2": 304.16, + "TestProxiedServerShow2B": 302.98, + "TestProxiedServerShow3": 276.05, + "TestProxiedServerShow3B": 261.07, + "TestProxiedServerShowB": 297.04, + "TestProxiedServerShowWatch": 164.7, + "TestProxiedServerStale": 6.27, + "TestProxiedServerState": 189.5, + "TestProxiedServerStatus": 40.45, + "TestProxiedServerStatuses": 2.26, + "TestProxiedServerStorageClass": 337.81, + "TestProxiedServerSwarm": 8.38, + "TestProxiedServerTag": 55.84, + "TestProxiedServerTodo": 96.61, + "TestProxiedServerTypes": 4.25, + "TestProxiedServerTypesAgreeWithCreate": 37.25, + "TestProxiedServerUnclaim": 16.93, + "TestProxiedServerUnclaimIfAssigneeParity": 35.25, + "TestProxiedServerUpdate": 306.84, + "TestProxiedServerUpdate2": 285.3, + "TestProxiedServerUpdate2B": 321.6, + "TestProxiedServerUpdate2C": 247.7, + "TestProxiedServerUpdate3": 375.23, + "TestProxiedServerUpdateB": 265.81, + "TestProxiedServerUpdateC": 265.62, + "TestProxiedServerUpdateClosePolicy": 82.95, + "TestProxiedServerUpdateConcurrentAppendNotes": 7.13, + "TestProxiedServerUpdateConcurrentClaim": 12.38, + "TestProxiedServerUpdateConcurrentMetadata": 11.36, + "TestProxiedServerUpdateHooks": 52.36, + "TestProxiedServerUpdateIfGuards": 83.41, + "TestProxiedServerUpdateWisp": 268.08, + "TestProxiedServerWispCreate": 78.86, + "TestProxiedServerWispCreateRejectsEnumViolatingVar": 13.68, + "TestProxiedServerWispGC": 164.43, + "TestProxiedServerWispList": 65.42, + "TestServerModeDeferAutoWake": 11.69, + "TestServerModeServe": 0.04, + "TestServerModeServeSkipsPostRunMaintenance": 13.74, + "TestServerModeSharedServeGlobalReportsTheServedDatabase": 34.13, + "TestServerModeTypesAgreeWithCreate": 36.55 } } diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go index 2bb020b41..9d1867f6b 100644 --- a/scripts/shard_manifest_test.go +++ b/scripts/shard_manifest_test.go @@ -220,7 +220,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { } } -// S3: the Bazel-only 30-shard block is not frozen like the legacy 15-shard +// S3: the Bazel-only 44-shard block is not frozen like the legacy 15-shard // block (TestShardScriptsListOnlyRealTests's B1 fix catches outright // corruption, but not a committed block that has drifted from the currently // discovered TestProxiedServer*/TestServerMode* test set, e.g. a test added, @@ -243,9 +243,10 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { func TestProxiedShardManifestGeneratorNotStale(t *testing.T) { python := requireHostTool(t, "python3") root := sourceRepoRoot(t) - cmd := exec.Command(python, "scripts/ci/gen_proxied_shard_manifest.py", "30", "--weights=duration", "--check") + shards := strconv.Itoa(bazelProxiedShardCount(t)) + cmd := exec.Command(python, "scripts/ci/gen_proxied_shard_manifest.py", shards, "--weights=duration", "--check") cmd.Dir = root if out, err := cmd.CombinedOutput(); err != nil { - t.Errorf("gen_proxied_shard_manifest.py 30 --weights=duration --check: %v\n%s", err, out) + t.Errorf("gen_proxied_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) } } From 470ec832b50b8e452a43f56c47e87128b8d2ecb7 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 14:56:32 +0000 Subject: [PATCH 015/151] feat(roles): LeaseReclaimer role, POST issues:reclaim, and bd reclaim on both routes (S9) (#7412) * feat(roles): LeaseReclaimer role and POST issues:reclaim (S9) Adds issueops.LeaseReclaimer.Reclaim(ReclaimRequest{Actor, OlderThan, Filter}) -> ReclaimResult{Reclaimed []ReclaimedLease{ID, PreviousOwner, Revision}}, the stale-lease sweep behind `bd reclaim`, on every leg: - dolt, embedded dolt and the unit-of-work provider all run one body, internal/storage/issueops.ExecuteReclaimInTx over ReclaimExpiredLeasesInTx, and record one version commit per sweep that reverted anything through the issue-operation funnel (so a deferred version commit on the context is honored); - each reverted row's fresh row_lock is published as its revision, so a former holder's guarded write fails with ErrVersionMismatch; - an id that is not a stale lease (absent, live, unheld) is left out of Reclaimed, never an error; the id scope is capped at 1000 (*TooManyReclaimIDsError, counted as sent); - the hook decorator fires on_update once per reverted row (and the proxied notifying provider records one update per row); telemetry wraps it like every role. bd serve publishes it as POST /v0/beads/issues:reclaim behind the new issues.reclaim capability (httpapi.Config.LeaseReclaimer, required like every role). Spec, apigen and the wire-shape golden are regenerated; the change is additive and wire_revision is unchanged. The HTTP client leg dials it (wire vocabulary, capability preflight, local validation in the role's order). Conformance: RunLeaseReclaimer* (12 cases: revert post-state, the revision race, live lease, grace window, id scope with absent/live ids, empty answer, assignee/label scopes re-homed from reclaim_scope.go, recovery event attribution, on_update once per row, refusals, the cap boundary, no request mutation) wired on dolt, embeddeddolt, uow and the served http leg, and so held by the leg lock. Handler tests cover the member mapping, the projection, every handler refusal by name and the role's refusals reaching the wire. Co-Authored-By: Claude Opus 5.5 * feat(bd): bd reclaim runs the LeaseReclaimer role on both routes (S9) `bd reclaim` takes issueops.LeaseReclaimer off the decorated store on the direct route (so a connected HTTP workspace dials issues:reclaim instead of refusing) and off the proxied provider's own accessor on the proxied route, replacing the raw ReclaimExpiredLeases calls and runReclaimProxiedServer. Visible changes: --json entries carry the reverted row's new `revision`, the workspace's on_update hook fires once per reverted row, and the version commit is the role's own ("bd: reclaim N expired lease(s)", honoring dolt.auto-commit through the context the way bd prune does). Tests: an embedded direct-route CLI test (scoped sweep with a live and an absent id, revision, one version commit, on_update once) and the proxied reclaim integration test now asserts the revision. Co-Authored-By: Claude Opus 5.5 * fix(roles): S9 review LOWs: one validator, field attribution, overflow bound - The library's ValidateReclaimRequest is the one definition of a valid reclaim: it now also refuses a blank assignee/label/label-any/exclude entry, and every refusal but the id cap is a *issueops.ReclaimFieldError naming its field. The httpapi handler no longer checks blank entries itself; failReclaimErr names the member the role's error is about (`ids` for the cap) instead of blaming `ids` for every validation. The HTTP client's restatement matches, pinned field by field. - older_than_seconds is refused at >= MaxInt64/1e9: that float rounds to 2^63 ns, which wrapped negative under the old > check. - New contract case RunLeaseReclaimerRefusesABlankScopeEntry on all four legs; handler tests for field attribution and the overflow boundary. - CHANGELOG names the 1000-id cap on bd reclaim --id and is rewrapped; the client's grace-window precision comment no longer claims exactness. Co-Authored-By: Claude Opus 5.5 * fix(roles): S9 review: regenerate the unsupported shell, cgo-tag the embedded contract, one actor spelling - go generate ./internal/httpclient: the new accessor moves the partial-shell trailer to 145 of 200. - lease_reclaimer_contract_test.go gets //go:build cgo like its siblings, so the pure-Go lane compiles. - The library trims the actor before attributing, matching the handler, so " alice " is attributed the same on every route. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(roles): S9 review minors: bound the reclaim actor on every leg, pin any_replica, validate before the uow Review minors on #7412, each in code this PR adds. - The reclaim actor lands on every reverted row's recovery event, in a VARCHAR(255) column, but ValidateReclaimRequest never bounded it: a 256-character actor passed validation on the dolt, embedded-dolt and unit-of-work legs and failed the sweep with the backend's own "Error 1105: string '...' is too large for column 'actor'" on the first row it would have reverted (all three legs answer exactly that with the bound below removed). The shared validator now holds the trimmed actor to types.MaxFieldLen, as claimIssueInTx does, so it is a *ReclaimFieldError for `actor` on every leg. The HTTP client restates the rule, and the parity test covers both sides of it. bd serve already refused it through the claim's name rules. The role's Actor and REFUSALS docs name the bound, and the conformance case runs it on all four legs. - ValidateReclaimRequest's doc no longer says bd serve's handler leaves every rule to it: it names the three wire rules the handler adds of its own. - TestReclaimSendsAnyReplicaOnlyWhenAsked pins the one filter member that widens a sweep: the client sends any_replica only when the caller set it. - The unit-of-work leg validates before opening its unit of work, as the dolt and embedded-dolt legs do; TestLeaseReclaimerRefusesBeforeOpeningAUOW pins it. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/scripts/embedded-cmd-test-shards.txt | 1 + .../scripts/embedded-storage-test-shards.txt | 1 + CHANGELOG.md | 15 + backend/conformance/BUILD.bazel | 2 + .../conformance/lease_reclaimer_contract.go | 625 ++++++++++++++++++ cmd/bd/BUILD.bazel | 1 + cmd/bd/reclaim.go | 76 ++- cmd/bd/reclaim_embedded_test.go | 133 ++++ cmd/bd/serve.go | 30 +- cmd/bd/serve_source_test.go | 27 + cmd/bd/serve_store_identity_test.go | 8 + cmd/bd/unclaim_proxied_integration_test.go | 6 + cmd/bd/unclaim_proxied_server.go | 31 - internal/httpapi/BUILD.bazel | 2 + internal/httpapi/apigen/types.gen.go | 43 +- internal/httpapi/claim.go | 7 + internal/httpapi/pinning.go | 2 + internal/httpapi/problem.go | 11 + internal/httpapi/reclaim.go | 249 +++++++ internal/httpapi/reclaim_test.go | 317 +++++++++ internal/httpapi/roles_test.go | 31 + internal/httpapi/routes.go | 11 + internal/httpapi/server.go | 25 +- internal/httpapi/spec/openapi.v0.yaml | 196 +++++- internal/httpapi/spec_parity_test.go | 11 +- .../httpapi/wireshape/testdata/golden.json | 87 +++ internal/httpclient/BUILD.bazel | 2 + internal/httpclient/accessors.go | 9 + internal/httpclient/encode/oracle_test.go | 2 + internal/httpclient/leasereclaimer.go | 135 ++++ internal/httpclient/served_harness_test.go | 2 + .../httpclient/served_lease_reclaimer_test.go | 111 ++++ internal/httpclient/served_test.go | 3 + internal/httpclient/unsupported_gen.go | 2 +- internal/httpclient/wave2b_roles_test.go | 86 +++ internal/httpclient/wire/handshake.go | 5 + internal/httpclient/wire/paths.go | 2 + .../wire/revision_tolerance_test.go | 4 + internal/httpclient/wire/skew_test.go | 11 +- internal/httpclient/wire/vocabulary_test.go | 1 + internal/storage/BUILD.bazel | 2 + internal/storage/dolt/BUILD.bazel | 2 + internal/storage/dolt/lease_reclaimer.go | 50 ++ .../dolt/lease_reclaimer_contract_test.go | 114 ++++ internal/storage/domain/db/issue.go | 7 + internal/storage/domain/issue.go | 17 + internal/storage/embeddeddolt/BUILD.bazel | 2 + .../storage/embeddeddolt/lease_reclaimer.go | 54 ++ .../lease_reclaimer_contract_test.go | 111 ++++ .../storage/externaldeps/uow_decorator.go | 6 +- internal/storage/hook_batch_applier_test.go | 4 + internal/storage/hook_decorator.go | 16 + internal/storage/hook_issue_operations.go | 1 + .../storage/hook_issue_operations_test.go | 3 + internal/storage/hook_lease_reclaimer.go | 42 ++ internal/storage/hook_lease_reclaimer_test.go | 83 +++ internal/storage/issueops/BUILD.bazel | 2 + internal/storage/issueops/lease.go | 9 +- internal/storage/issueops/lease_reclaimer.go | 123 ++++ .../storage/issueops/lease_reclaimer_test.go | 107 +++ .../storage/role_accessor_decorator_test.go | 130 ++-- internal/storage/storage.go | 9 + internal/storage/uow/BUILD.bazel | 2 + internal/storage/uow/lease_reclaimer.go | 60 ++ .../uow/lease_reclaimer_contract_test.go | 146 ++++ internal/storage/uow/notifying.go | 24 + internal/telemetry/BUILD.bazel | 1 + internal/telemetry/lease_reclaimer.go | 36 + .../telemetry/role_accessor_decorator_test.go | 12 +- internal/types/types.go | 7 + issue_roles_external_test.go | 33 + issueops/BUILD.bazel | 1 + issueops/leasereclaimer.go | 183 +++++ 73 files changed, 3582 insertions(+), 140 deletions(-) create mode 100644 backend/conformance/lease_reclaimer_contract.go create mode 100644 cmd/bd/reclaim_embedded_test.go create mode 100644 internal/httpapi/reclaim.go create mode 100644 internal/httpapi/reclaim_test.go create mode 100644 internal/httpclient/leasereclaimer.go create mode 100644 internal/httpclient/served_lease_reclaimer_test.go create mode 100644 internal/storage/dolt/lease_reclaimer.go create mode 100644 internal/storage/dolt/lease_reclaimer_contract_test.go create mode 100644 internal/storage/embeddeddolt/lease_reclaimer.go create mode 100644 internal/storage/embeddeddolt/lease_reclaimer_contract_test.go create mode 100644 internal/storage/hook_lease_reclaimer.go create mode 100644 internal/storage/hook_lease_reclaimer_test.go create mode 100644 internal/storage/issueops/lease_reclaimer.go create mode 100644 internal/storage/issueops/lease_reclaimer_test.go create mode 100644 internal/storage/uow/lease_reclaimer.go create mode 100644 internal/storage/uow/lease_reclaimer_contract_test.go create mode 100644 internal/telemetry/lease_reclaimer.go create mode 100644 issueops/leasereclaimer.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 7a1c26452..a0ef80ddb 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -346,6 +346,7 @@ 50 24 TestEmbeddedCloseBasic 50 24 TestEmbeddedImportUpsertAndPrefixSync 50 24 TestEmbeddedQuery +50 24 TestEmbeddedReclaimRunsTheLeaseReclaimerRole 50 24 TestEmbeddedUpdateRoutedStoreCommitsTargetHead 50 25 TestEmbeddedBackupConcurrent diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index c92319f31..da78fa452 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -433,6 +433,7 @@ 20 15 TestExpectedRevisionContract 20 15 TestLargeBatchApplyStatementCounts356_Embedded 20 15 TestLargeGraphCreateTiming_Embedded +20 15 TestLeaseReclaimerContract 20 15 TestReadyWorkPageTotalFilterMatrix 20 15 TestReclaimExpiredLeaseSurvivesRestartEmbedded 20 15 TestRemoteEntryPointsUseVersionControlOps diff --git a/CHANGELOG.md b/CHANGELOG.md index 9831b832c..3497db5d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `issueops.LeaseReclaimer`, the stale-lease sweep behind `bd reclaim`, is a + role on every leg (dolt, embedded dolt, the unit-of-work provider and the + HTTP client), and `bd serve` publishes it as `POST /v0/beads/issues:reclaim` + behind the new `issues.reclaim` capability (additive; `wire_revision` is + unchanged). Every reverted row mints a new revision, reported per entry as + `revision`, so a holder that writes with its pre-reclaim token gets a version + mismatch. An id that is not a stale lease is left out of the answer, not + refused. `bd reclaim` now runs the role on both routes, which changes four + things a script can see: its `--json` entries gain `revision`; the + workspace's `on_update` hook fires once per reverted row (the raw path fired + none); `--id` now accepts at most 1000 ids per run (more is refused, never + truncated); and the role records its own version commit, honoring + `dolt.auto-commit` the way `bd prune` does, as + `bd: reclaim N expired lease(s)` on both routes. Over a connected HTTP + workspace `bd reclaim` now works instead of refusing. - `bd create --graph` now plans its batch through `issueops.BatchApplier` instead of the old `buildDomainGraphPlan` path, so a graph create gets the same atomic multi-row semantics as `bd batch apply`. A `waits-for` edge's diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index 1f9632e79..f5c8882cc 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -35,6 +35,7 @@ go_library( "issue_operations_contract.go", "issue_operations_staging.go", "journal_contract.go", + "lease_reclaimer_contract.go", "lifecycle_close_reopen_contract.go", "lifecycle_create_contract.go", "lifecycle_update_contract.go", @@ -62,6 +63,7 @@ go_library( importpath = "github.com/steveyegge/beads/backend/conformance", visibility = ["//visibility:public"], deps = [ + "//internal/hooks", "//internal/storage", "//internal/storage/issueops", "//internal/storage/kvkeys", diff --git a/backend/conformance/lease_reclaimer_contract.go b/backend/conformance/lease_reclaimer_contract.go new file mode 100644 index 000000000..eadc6c176 --- /dev/null +++ b/backend/conformance/lease_reclaimer_contract.go @@ -0,0 +1,625 @@ +package conformance + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "slices" + "sort" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/hooks" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// This file holds the contract every implementation of publicops.LeaseReclaimer +// must satisfy. Each case asserts what issueops/leasereclaimer.go PROMISES, +// cited by symbol. +// +// FOUR LEGS, ONE BODY. dolt, embedded dolt and the unit-of-work provider all +// reach internal/storage/issueops.ExecuteReclaimInTx, and the http leg reaches +// the same body on the far side of POST /v0/beads/issues:reclaim. So a per-leg +// failure here is a wrapper's failure — a lost scope, a revision that never +// reached the caller, a refusal that stopped matching errors.Is — and the +// cases are written to catch exactly those. +// +// EVERY SWEEP HERE IS SCOPED. The legs share one database per suite with every +// other role's cases, and an UNSCOPED reclaim would revert whatever stale lease +// another case left behind. Every call below names its ids, or an assignee or +// label this case alone uses, so a case can never reach outside what it seeded. +// The workspace-wide form is the same SQL with the scope clause absent, which +// sqlbuild.ReclaimScopeSQL's own tests pin. +// +// STALENESS IS MANUFACTURED, NOT WAITED FOR. The role refuses a negative +// OlderThan (the raw method's -time.Hour trick is exactly what it rules out), +// so a case claims through the leg's real Claimer — which writes the lease row +// every reclaim reads — and then moves that row's lease_expires_at into the +// past through Exec. +type LeaseReclaimerFixture struct { + // IssuePrefix namespaces the ids each assertion seeds, so several of them + // can share one database. + IssuePrefix string + // LeaseReclaimer is the surface under test. + LeaseReclaimer publicops.LeaseReclaimer + // CreateIssue seeds a durable issue in the issues plane. + CreateIssue func(context.Context, *types.Issue, string) error + // Claimer takes the claim, and with it the lease row, a reclaim reverts. It + // is the leg's own role rather than a raw insert so the lease is the shape + // a real claim leaves, granted_node included. + Claimer publicops.Claimer + // Lifecycle performs the guarded write the revision case races against the + // reclaim. A nil Lifecycle SKIPS that case loudly. + Lifecycle publicops.Lifecycle + // Exec runs raw statements against the backing database: the cases use it + // to age a lease and to attach labels. A nil Exec SKIPS every case that + // needs a stale lease, which is nearly all of them, so every leg supplies + // one. + Exec func(ctx context.Context, statements []SQLStatement) error + // QueryScalar runs a single-row query and scans it, so the post-state is + // read RAW rather than through the role being tested. + QueryScalar func(context.Context, string, []any, ...any) error + // HookedLeaseReclaimer is the same role taken off the leg's HOOK-FIRING + // chain (storage.HookFiringStore on the store legs and the http leg, + // uow.NewNotifyingProvider on the unit-of-work leg), built over the runner + // FiredUpdates reads. Nil, with FiredUpdates, SKIPS the hook case loudly. + HookedLeaseReclaimer publicops.LeaseReclaimer + // FiredUpdates reports, in firing order, the issue id of every on_update + // the hooked chain ran so far. It must not return before the hooks it is + // asked about have finished. + FiredUpdates func(context.Context) ([]string, error) +} + +const ( + leaseReclaimerReaper = "reaper" + leaseReclaimerHolder = "lr-holder" +) + +// RunLeaseReclaimerRevertsAStaleLease pins the post-state the role names +// (issueops/leasereclaimer.go, LeaseReclaimer: "each reclaimed row moves +// in_progress -> open and the assignee is cleared") and the three members of +// the ReclaimedLease it reports, each READ RAW. +func RunLeaseReclaimerRevertsAStaleLease(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-revert" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id) + + result := leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{id}}, + }) + if len(result.Reclaimed) != 1 { + t.Fatalf("Reclaimed = %+v, want exactly %s", result.Reclaimed, id) + } + got := result.Reclaimed[0] + if got.ID != id || got.PreviousOwner != leaseReclaimerHolder { + t.Fatalf("Reclaimed[0] = %+v, want {ID:%s PreviousOwner:%s}", got, id, leaseReclaimerHolder) + } + + if status := leaseReclaimerScalar[string](t, ctx, fixture, "SELECT status FROM issues WHERE id = ?", id); status != string(types.StatusOpen) { + t.Fatalf("status after reclaim = %q, want %q", status, types.StatusOpen) + } + if held := leaseReclaimerScalar[int](t, ctx, fixture, + "SELECT COUNT(*) FROM issues WHERE id = ? AND (COALESCE(assignee, '') <> '' OR started_at IS NOT NULL)", id); held != 0 { + t.Fatal("reclaim left the assignee or started_at set; the row is still held by a dead worker") + } + if leases := leaseReclaimerScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", id); leases != 0 { + t.Fatalf("lease rows after reclaim = %d, want 0", leases) + } + stored := leaseReclaimerScalar[int64](t, ctx, fixture, "SELECT row_lock FROM issues WHERE id = ?", id) + if got.Revision != types.RevisionToken(stored) { + t.Fatalf("Reclaimed[0].Revision = %q, want the row's own token %q", got.Revision, types.RevisionToken(stored)) + } +} + +// RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder is the reason the +// revision is minted at all (LeaseReclaimer: "a racing holder's guarded write +// fails with ErrVersionMismatch"). A worker that lost its lease and wakes up +// holding the token its claim handed it must NOT be able to land a guarded +// write over the issue another worker may already have taken; the token the +// reclaim reported, and only it, lands. +func RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + if fixture.Lifecycle == nil { + t.Skip("LeaseReclaimerFixture.Lifecycle is nil: this backend cannot race a guarded write against the reclaim") + } + id := fixture.IssuePrefix + "-fence" + claimed := leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id)[id] + + result := leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{id}}, + }) + if len(result.Reclaimed) != 1 { + t.Fatalf("Reclaimed = %+v, want exactly %s", result.Reclaimed, id) + } + minted, err := types.ParseRevisionToken(result.Reclaimed[0].Revision) + if err != nil { + t.Fatalf("Reclaimed[0].Revision %q is not a revision token: %v", result.Reclaimed[0].Revision, err) + } + if minted == claimed { + t.Fatalf("reclaim reported revision %d, the claim's own token: nothing was minted", minted) + } + + stale := claimed + _, err = fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: leaseReclaimerHolder, IssueID: id, ExpectedVersion: &stale, + Patch: publicops.IssuePatch{Notes: publicops.Field[string]{Set: true, Value: "late write"}}, + }) + if !errors.Is(err, publicops.ErrVersionMismatch) { + t.Fatalf("guarded write with the pre-reclaim token = %v, want ErrVersionMismatch", err) + } + + fresh := minted + if _, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: leaseReclaimerReaper, IssueID: id, ExpectedVersion: &fresh, + Patch: publicops.IssuePatch{Notes: publicops.Field[string]{Set: true, Value: "after reclaim"}}, + }); err != nil { + t.Fatalf("guarded write with the token the reclaim reported = %v, want it to land", err) + } +} + +// RunLeaseReclaimerLeavesALiveLeaseAlone pins staleness as the gate: a lease +// that has not expired is not eligible, even when its id is named, and naming +// it is not an error (ReclaimRequest.Filter: "FILTER.IDS IS NOT AN ERROR WHEN +// IT NAMES A NON-STALE OR UNKNOWN ISSUE"). +func RunLeaseReclaimerLeavesALiveLeaseAlone(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-live" + leaseReclaimerSeedClaimed(t, ctx, fixture, leaseReclaimerHolder, id) + + result := leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{id}}, + }) + if len(result.Reclaimed) != 0 { + t.Fatalf("Reclaimed = %+v, want nothing: the lease has not expired", result.Reclaimed) + } + leaseReclaimerRequireHeld(t, ctx, fixture, id) +} + +// RunLeaseReclaimerHonorsTheGraceWindow pins ReclaimRequest.OlderThan: a lease +// expired less than OlderThan ago is left alone, and the same lease is +// reclaimed once the window is narrower than its age. +func RunLeaseReclaimerHonorsTheGraceWindow(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-grace" + leaseReclaimerSeedClaimed(t, ctx, fixture, leaseReclaimerHolder, id) + leaseReclaimerAge(t, ctx, fixture, 10*time.Minute, id) + + request := publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + OlderThan: time.Hour, + Filter: publicops.ReclaimFilter{IDs: []string{id}}, + } + if result := leaseReclaim(t, ctx, fixture, request); len(result.Reclaimed) != 0 { + t.Fatalf("OlderThan 1h over a lease 10m stale: Reclaimed = %+v, want nothing", result.Reclaimed) + } + leaseReclaimerRequireHeld(t, ctx, fixture, id) + + request.OlderThan = time.Minute + if got := leaseReclaimerIDs(leaseReclaim(t, ctx, fixture, request)); !slices.Equal(got, []string{id}) { + t.Fatalf("OlderThan 1m over a lease 10m stale: Reclaimed = %v, want [%s]", got, id) + } +} + +// RunLeaseReclaimerScopesToTheNamedIDs pins the IDs scope from both sides: an +// id that is not named is never reverted though it is stale, and an id that is +// named but absent or live is simply missing from Reclaimed — the call does +// not fail and does not report it. +func RunLeaseReclaimerScopesToTheNamedIDs(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + named := fixture.IssuePrefix + "-scope-named" + unnamed := fixture.IssuePrefix + "-scope-unnamed" + live := fixture.IssuePrefix + "-scope-live" + ghost := fixture.IssuePrefix + "-scope-ghost" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, named, unnamed) + leaseReclaimerSeedClaimed(t, ctx, fixture, leaseReclaimerHolder, live) + + result := leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{ghost, live, named}}, + }) + if got := leaseReclaimerIDs(result); !slices.Equal(got, []string{named}) { + t.Fatalf("Reclaimed = %v, want [%s]: only the named stale lease", got, named) + } + leaseReclaimerRequireHeld(t, ctx, fixture, unnamed) + leaseReclaimerRequireHeld(t, ctx, fixture, live) +} + +// RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult pins the empty answer: +// a sweep scoped only to ids that name no row is not a refusal, and its +// Reclaimed is empty and NON-NIL (ReclaimResult.Reclaimed: "Never nil for a +// successful call"). +func RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + result, err := fixture.LeaseReclaimer.Reclaim(ctx, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{fixture.IssuePrefix + "-absent-a", fixture.IssuePrefix + "-absent-b"}}, + }) + if err != nil { + t.Fatalf("Reclaim scoped to absent ids = %v, want an empty answer", err) + } + if result.Reclaimed == nil { + t.Fatal("Reclaimed is nil; the role promises an empty, non-nil slice") + } + if len(result.Reclaimed) != 0 { + t.Fatalf("Reclaimed = %+v, want nothing", result.Reclaimed) + } +} + +// RunLeaseReclaimerScopesToAssigneesAndLabels re-homes the raw method's scope +// cases (testReclaimScoped) onto the role: assignee, label (AND), label-any +// (OR) and exclude-label each narrow the sweep and never widen it. +func RunLeaseReclaimerScopesToAssigneesAndLabels(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + p := fixture.IssuePrefix + a1, a2, b1, c1 := p+"-lbl-a1", p+"-lbl-a2", p+"-lbl-b1", p+"-lbl-c1" + laneA, laneB, opus := p+"-lane-a", p+"-lane-b", p+"-tier-opus" + workerA, workerB, workerC := p+"-wa", p+"-wb", p+"-wc" + leaseReclaimerSeedStale(t, ctx, fixture, workerA, a1, a2) + leaseReclaimerSeedStale(t, ctx, fixture, workerB, b1) + leaseReclaimerSeedStale(t, ctx, fixture, workerC, c1) + leaseReclaimerLabel(t, ctx, fixture, a1, laneA) + leaseReclaimerLabel(t, ctx, fixture, a2, laneA) + leaseReclaimerLabel(t, ctx, fixture, a2, opus) + leaseReclaimerLabel(t, ctx, fixture, b1, laneB) + + reclaim := func(filter publicops.ReclaimFilter) []string { + t.Helper() + return leaseReclaimerIDs(leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{Actor: leaseReclaimerReaper, Filter: filter})) + } + + if got := reclaim(publicops.ReclaimFilter{Labels: []string{laneB}}); !slices.Equal(got, []string{b1}) { + t.Fatalf("reclaim(labels %s) = %v, want [%s]", laneB, got, b1) + } + if got := reclaim(publicops.ReclaimFilter{Assignees: []string{workerC}}); !slices.Equal(got, []string{c1}) { + t.Fatalf("reclaim(assignees %s) = %v, want [%s]", workerC, got, c1) + } + if got := reclaim(publicops.ReclaimFilter{Labels: []string{laneA}, ExcludeLabels: []string{opus}}); !slices.Equal(got, []string{a1}) { + t.Fatalf("reclaim(labels %s, exclude %s) = %v, want [%s]", laneA, opus, got, a1) + } + leaseReclaimerRequireHeld(t, ctx, fixture, a2) + if got := reclaim(publicops.ReclaimFilter{LabelsAny: []string{p + "-lane-z", opus}}); !slices.Equal(got, []string{a2}) { + t.Fatalf("reclaim(labels-any lane-z,%s) = %v, want [%s]", opus, got, a2) + } +} + +// RunLeaseReclaimerAttributesTheRecoveryEvent pins ReclaimRequest.Actor: the +// recovery event each reverted row records names the actor who ran the sweep, +// with the lease's former holder beside it. +func RunLeaseReclaimerAttributesTheRecoveryEvent(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-event" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id) + leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{id}}, + }) + if n := leaseReclaimerScalar[int](t, ctx, fixture, + "SELECT COUNT(*) FROM events WHERE issue_id = ? AND event_type = ? AND actor = ? AND old_value = ?", + id, string(types.EventLeaseReclaimed), leaseReclaimerReaper, leaseReclaimerHolder); n != 1 { + t.Fatalf("lease_reclaimed events by %s naming %s = %d, want 1", leaseReclaimerReaper, leaseReclaimerHolder, n) + } +} + +// RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow pins the hook the role +// names (LeaseReclaimer: "ITS HOOK IS on_update, once per reverted row"): each +// reverted row fires on_update exactly once, and a row the sweep did not +// revert — named but live — fires nothing. +func RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + if fixture.HookedLeaseReclaimer == nil || fixture.FiredUpdates == nil { + t.Skip("LeaseReclaimerFixture.HookedLeaseReclaimer/FiredUpdates is nil: this backend cannot observe its hooks") + } + first := fixture.IssuePrefix + "-hook-a" + second := fixture.IssuePrefix + "-hook-b" + live := fixture.IssuePrefix + "-hook-live" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, first, second) + leaseReclaimerSeedClaimed(t, ctx, fixture, leaseReclaimerHolder, live) + before, err := fixture.FiredUpdates(ctx) + if err != nil { + t.Fatalf("FiredUpdates before: %v", err) + } + + result, err := fixture.HookedLeaseReclaimer.Reclaim(ctx, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{first, second, live}}, + }) + if err != nil { + t.Fatalf("hooked Reclaim: %v", err) + } + if got := leaseReclaimerIDs(result); !slices.Equal(got, []string{first, second}) { + t.Fatalf("Reclaimed = %v, want [%s %s]", got, first, second) + } + + after, err := fixture.FiredUpdates(ctx) + if err != nil { + t.Fatalf("FiredUpdates after: %v", err) + } + fired := map[string]int{} + for _, id := range after[len(before):] { + fired[id]++ + } + for _, id := range []string{first, second} { + if fired[id] != 1 { + t.Fatalf("on_update fired %d time(s) for %s, want exactly 1 (fired: %v)", fired[id], id, after[len(before):]) + } + } + if fired[live] != 0 { + t.Fatalf("on_update fired for %s, which the sweep did not revert", live) + } +} + +// RunLeaseReclaimerRefusesAMalformedRequest pins every refusal the role lists, +// each as ErrValidation (the cap as *TooManyReclaimIDsError), and that a +// refusal changes nothing: the stale lease every request names survives all of +// them. +func RunLeaseReclaimerRefusesAMalformedRequest(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-malformed" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id) + + tooMany := make([]string, publicops.MaxReclaimIDs+1) + for i := range tooMany { + tooMany[i] = id + } + for _, test := range []struct { + name string + request publicops.ReclaimRequest + tooMany bool + }{ + {"an empty actor", publicops.ReclaimRequest{Filter: publicops.ReclaimFilter{IDs: []string{id}}}, false}, + // One past the column every reverted row's recovery event records it + // in: a typed refusal before the sweep, never the backend's own error + // from the first row it would have reverted. + {"an over-long actor", publicops.ReclaimRequest{Actor: strings.Repeat("r", types.MaxFieldLen+1), + Filter: publicops.ReclaimFilter{IDs: []string{id}}}, false}, + {"a negative grace window", publicops.ReclaimRequest{Actor: leaseReclaimerReaper, OlderThan: -time.Second, + Filter: publicops.ReclaimFilter{IDs: []string{id}}}, false}, + {"a blank scoped id", publicops.ReclaimRequest{Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{id, ""}}}, false}, + {"more ids than the cap", publicops.ReclaimRequest{Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: tooMany}}, true}, + } { + _, err := fixture.LeaseReclaimer.Reclaim(ctx, test.request) + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("%s: Reclaim error = %v, want ErrValidation", test.name, err) + } + if test.tooMany { + var capErr *publicops.TooManyReclaimIDsError + if !errors.As(err, &capErr) || capErr.Requested != len(tooMany) || capErr.Cap != publicops.MaxReclaimIDs { + t.Fatalf("%s: Reclaim error = %v, want *TooManyReclaimIDsError{Requested:%d Cap:%d}", + test.name, err, len(tooMany), publicops.MaxReclaimIDs) + } + } + leaseReclaimerRequireHeld(t, ctx, fixture, id) + } +} + +// RunLeaseReclaimerRefusesABlankScopeEntry pins the blank-entry refusal on +// the scopes that are not ids: an empty or all-blank assignee or label is +// ErrValidation as a *ReclaimFieldError naming its field, and changes +// nothing — the stale lease every request names survives. +func RunLeaseReclaimerRefusesABlankScopeEntry(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-blankscope" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id) + + for _, test := range []struct { + field string + filter publicops.ReclaimFilter + }{ + {publicops.ReclaimFieldAssignees, publicops.ReclaimFilter{IDs: []string{id}, Assignees: []string{leaseReclaimerHolder, " "}}}, + {publicops.ReclaimFieldLabels, publicops.ReclaimFilter{IDs: []string{id}, Labels: []string{""}}}, + {publicops.ReclaimFieldLabelsAny, publicops.ReclaimFilter{IDs: []string{id}, LabelsAny: []string{"\t"}}}, + {publicops.ReclaimFieldExcludeLabels, publicops.ReclaimFilter{IDs: []string{id}, ExcludeLabels: []string{"x", " "}}}, + } { + _, err := fixture.LeaseReclaimer.Reclaim(ctx, publicops.ReclaimRequest{Actor: leaseReclaimerReaper, Filter: test.filter}) + var fieldErr *publicops.ReclaimFieldError + if !errors.Is(err, publicops.ErrValidation) || !errors.As(err, &fieldErr) || fieldErr.Field != test.field { + t.Fatalf("blank %s entry: Reclaim error = %v, want a *ReclaimFieldError naming %s", test.field, err, test.field) + } + leaseReclaimerRequireHeld(t, ctx, fixture, id) + } +} + +// RunLeaseReclaimerAcceptsExactlyTheCap pins the cap's boundary: MaxReclaimIDs +// entries is a request, one more is a refusal (the case above). +func RunLeaseReclaimerAcceptsExactlyTheCap(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + ids := make([]string, publicops.MaxReclaimIDs) + for i := range ids { + ids[i] = fmt.Sprintf("%s-cap-%04d", fixture.IssuePrefix, i) + } + result, err := fixture.LeaseReclaimer.Reclaim(ctx, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, Filter: publicops.ReclaimFilter{IDs: ids}, + }) + if err != nil { + t.Fatalf("Reclaim with exactly MaxReclaimIDs ids = %v, want an answer", err) + } + if len(result.Reclaimed) != 0 { + t.Fatalf("Reclaimed = %+v, want nothing: none of the ids exist", result.Reclaimed) + } +} + +// RunLeaseReclaimerDoesNotMutateTheCallerRequest pins the role's "never mutate +// caller-owned request values": the scope slices come back as they were sent. +func RunLeaseReclaimerDoesNotMutateTheCallerRequest(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-nomut" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id) + ids := []string{fixture.IssuePrefix + "-nomut-ghost", id} + assignees := []string{leaseReclaimerHolder} + request := publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: ids, Assignees: assignees}, + } + leaseReclaim(t, ctx, fixture, request) + if !slices.Equal(ids, []string{fixture.IssuePrefix + "-nomut-ghost", id}) || !slices.Equal(assignees, []string{leaseReclaimerHolder}) { + t.Fatalf("Reclaim rewrote the caller's scope: ids=%v assignees=%v", ids, assignees) + } +} + +// NewUpdateHookLog plants an on_update script that appends the id of every +// issue it fires for, and returns the runner a hook-firing chain is built over +// plus a reader for LeaseReclaimerFixture.FiredUpdates. The reader waits for +// in-flight hooks before reading, so it answers for every hook already fired. +// +// It answers (nil, nil) where a shell hook cannot run, so a fixture built from +// it leaves the hook case to skip rather than fail. +func NewUpdateHookLog(t *testing.T) (*hooks.Runner, func(context.Context) ([]string, error)) { + t.Helper() + if runtime.GOOS == "windows" { + return nil, nil + } + dir := t.TempDir() + log := filepath.Join(t.TempDir(), "on_update.log") + script := "#!/bin/sh\necho \"$1\" >> " + log + "\n" + if err := os.WriteFile(filepath.Join(dir, "on_update"), []byte(script), 0o755); err != nil { //nolint:gosec // G306: a hook must be executable to run at all + t.Fatalf("plant the on_update hook: %v", err) + } + runner := hooks.NewRunner(dir) + return runner, func(context.Context) ([]string, error) { + if !runner.Wait(30 * time.Second) { + return nil, errors.New("on_update hooks still running after 30s") + } + data, err := os.ReadFile(log) // #nosec G304 -- this helper's own temp file + if errors.Is(err, os.ErrNotExist) { + return []string{}, nil + } + if err != nil { + return nil, err + } + var ids []string + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line = strings.TrimSpace(line); line != "" { + ids = append(ids, line) + } + } + return ids, nil + } +} + +func requireLeaseReclaimerExec(t *testing.T, fixture LeaseReclaimerFixture) { + t.Helper() + if fixture.Exec == nil || fixture.Claimer == nil || fixture.QueryScalar == nil { + t.Skip("LeaseReclaimerFixture.Exec/Claimer/QueryScalar is nil: this backend cannot manufacture a stale lease") + } +} + +// leaseReclaimerSeedClaimed creates each id and claims it as holder through the +// leg's Claimer, answering each claim's row version. +func leaseReclaimerSeedClaimed(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, holder string, ids ...string) map[string]int64 { + t.Helper() + versions := map[string]int64{} + for _, id := range ids { + if err := fixture.CreateIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + claimed, err := fixture.Claimer.Claim(ctx, publicops.ClaimRequest{Actor: holder, IssueID: id}) + if err != nil { + t.Fatalf("claim %s as %s: %v", id, holder, err) + } + if claimed.Issue == nil { + t.Fatalf("claim %s answered no issue", id) + } + versions[id] = claimed.Issue.RowVersion + } + return versions +} + +// leaseReclaimerSeedStale claims each id and ages its lease an hour past +// expiry, far beyond any grace window a case below uses by default. +func leaseReclaimerSeedStale(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, holder string, ids ...string) map[string]int64 { + t.Helper() + versions := leaseReclaimerSeedClaimed(t, ctx, fixture, holder, ids...) + leaseReclaimerAge(t, ctx, fixture, time.Hour, ids...) + return versions +} + +// leaseReclaimerAge moves each id's lease expiry to age in the past. +func leaseReclaimerAge(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, age time.Duration, ids ...string) { + t.Helper() + expired := time.Now().UTC().Add(-age) + statements := make([]SQLStatement, 0, len(ids)) + for _, id := range ids { + statements = append(statements, SQLStatement{ + Query: "UPDATE leases SET lease_expires_at = ? WHERE issue_id = ?", + Args: []any{expired, id}, + }) + } + if err := fixture.Exec(ctx, statements); err != nil { + t.Fatalf("age leases %v: %v", ids, err) + } + for _, id := range ids { + if n := leaseReclaimerScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", id); n != 1 { + t.Fatalf("lease rows for %s = %d, want 1: the claim wrote no lease to reclaim", id, n) + } + } +} + +func leaseReclaimerLabel(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, id, label string) { + t.Helper() + if err := fixture.Exec(ctx, []SQLStatement{{ + Query: "INSERT INTO labels (issue_id, label) VALUES (?, ?)", + Args: []any{id, label}, + }}); err != nil { + t.Fatalf("label %s %s: %v", id, label, err) + } +} + +func leaseReclaimerRequireHeld(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, id string) { + t.Helper() + if status := leaseReclaimerScalar[string](t, ctx, fixture, "SELECT status FROM issues WHERE id = ?", id); status != string(types.StatusInProgress) { + t.Fatalf("%s status = %q, want it still %q", id, status, types.StatusInProgress) + } +} + +func leaseReclaim(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, request publicops.ReclaimRequest) publicops.ReclaimResult { + t.Helper() + result, err := fixture.LeaseReclaimer.Reclaim(ctx, request) + if err != nil { + t.Fatalf("Reclaim(%+v): %v", request.Filter, err) + } + if result.Reclaimed == nil { + t.Fatal("Reclaimed is nil; the role promises a non-nil slice for a successful call") + } + return result +} + +func leaseReclaimerIDs(result publicops.ReclaimResult) []string { + ids := make([]string, 0, len(result.Reclaimed)) + for _, r := range result.Reclaimed { + ids = append(ids, r.ID) + } + sort.Strings(ids) + return ids +} + +func leaseReclaimerScalar[T any](t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, query string, args ...any) T { + t.Helper() + var value T + if err := fixture.QueryScalar(ctx, query, args, &value); err != nil { + t.Fatalf("%s %v: %v", query, args, err) + } + return value +} diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index b5e03704f..309ae4fc3 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1048,6 +1048,7 @@ go_test( "ready_role_request_test.go", "ready_test.go", "ready_wisp_proxied_integration_test.go", + "reclaim_embedded_test.go", "reclaim_test.go", "recompute_blocked_proxied_integration_test.go", "registered_backend_workspace_paths_test.go", diff --git a/cmd/bd/reclaim.go b/cmd/bd/reclaim.go index 2a5262d99..7bde1f5b8 100644 --- a/cmd/bd/reclaim.go +++ b/cmd/bd/reclaim.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "strings" @@ -8,9 +9,11 @@ import ( "github.com/spf13/pflag" "github.com/steveyegge/beads/internal/metrics" - "github.com/steveyegge/beads/internal/storage/issueops" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) var reclaimCmd = &cobra.Command{ @@ -99,31 +102,68 @@ Examples: CheckReadonly("reclaim") - if usesProxiedServer() { - return runReclaimProxiedServer(rootCtx, olderThan, filter) - } - - ctx := rootCtx - reclaimed, err := store.ReclaimExpiredLeases(ctx, olderThan, filter, currentActor()) + reclaimer, err := openLeaseReclaimer() if err != nil { return HandleErrorRespectJSON("reclaim: %v", err) } - - ids := make([]string, 0, len(reclaimed)) - for _, r := range reclaimed { - ids = append(ids, r.ID) + // The role mints the sweep's version commit itself, so batch and off + // modes have to be said on the CONTEXT, the way `bd prune` says them. + // The proxied route applied the policy to rootCtx in the root pre-run + // already, and issueOpsContext leaves a context that defers alone. + ctx := rootCtx + if !usesProxiedServer() { + if ctx, err = issueOpsContext(rootCtx); err != nil { + return HandleErrorRespectJSON("%v", err) + } } - if err := commitPendingIfEmbedded(ctx, store, currentActor(), doltAutoCommitParams{ - Command: "reclaim", - IssueIDs: ids, - }); err != nil { - return HandleErrorRespectJSON("failed to commit: %v", err) + result, err := reclaimer.Reclaim(ctx, issueops.ReclaimRequest{ + Actor: currentActor(), + OlderThan: olderThan, + Filter: filter, + }) + if err != nil { + return HandleErrorRespectJSON("reclaim: %v", err) + } + if len(result.Reclaimed) > 0 { + commandDidWrite.Store(true) } - return renderReclaim(reclaimed, !filter.IsEmpty()) + return renderReclaim(result.Reclaimed, !filter.IsEmpty()) }, } +// openLeaseReclaimer hands back the stale-lease sweep for whichever route this +// invocation is on: the store's own accessor (through every decorator the store +// was wired with, so the workspace's on_update hook fires once per reverted +// row, and over a registered HTTP backend the wire's reclaimIssues), or the +// proxied-server provider's. +func openLeaseReclaimer() (issueops.LeaseReclaimer, error) { + if usesProxiedServer() { + return proxiedLeaseReclaimer() + } + if store == nil { + if err := ensureStoreActive(); err != nil { + return nil, err + } + } + return store.LeaseReclaimer() +} + +// proxiedLeaseReclaimer takes the sweep through the proxied-server provider's +// OWN capability accessor, proxiedSweeper's two-step and for its reason: the +// accessor is where each layer (the notifying provider's hooks included) is +// added. +func proxiedLeaseReclaimer() (issueops.LeaseReclaimer, error) { + if uowProvider == nil { + return nil, errors.New("proxied-server UOW provider not initialized") + } + src, ok := uowProvider.(uow.LeaseReclaimerSource) + if !ok { + return nil, fmt.Errorf("proxied-server provider %T does not offer the lease-sweep surface", uowProvider) + } + return src.LeaseReclaimer() +} + func renderReclaim(reclaimed []types.ReclaimedLease, scoped bool) error { if jsonOutput { return outputJSON(map[string]interface{}{ @@ -224,7 +264,7 @@ func reclaimFilterFromFlags(cmd *cobra.Command) (types.ReclaimFilter, error) { } func init() { - reclaimCmd.Flags().Duration("older-than", 2*issueops.DefaultLeaseTTL, + reclaimCmd.Flags().Duration("older-than", 2*storeops.DefaultLeaseTTL, "Only reclaim leases that expired at least this long ago (grace window)") registerReclaimScopeFlags(reclaimCmd.Flags()) rootCmd.AddCommand(reclaimCmd) diff --git a/cmd/bd/reclaim_embedded_test.go b/cmd/bd/reclaim_embedded_test.go new file mode 100644 index 000000000..9a89f0961 --- /dev/null +++ b/cmd/bd/reclaim_embedded_test.go @@ -0,0 +1,133 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/types" +) + +// TestEmbeddedReclaimRunsTheLeaseReclaimerRole drives `bd reclaim` on the +// direct route, where it now runs issueops.LeaseReclaimer off the decorated +// store: the stale lease it names is reverted with a fresh revision, a live +// lease and an absent id it names are left out without an error, the sweep +// records its own version commit, and the workspace's on_update hook fires once +// for the reverted row. +func TestEmbeddedReclaimRunsTheLeaseReclaimerRole(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "rc") + + hookLog := filepath.Join(t.TempDir(), "on_update.log") + hooksDir := filepath.Join(beadsDir, "hooks") + if err := os.MkdirAll(hooksDir, 0o755); err != nil { + t.Fatalf("mkdir hooks: %v", err) + } + script := "#!/bin/sh\necho \"$1\" >> " + hookLog + "\n" + if err := os.WriteFile(filepath.Join(hooksDir, "on_update"), []byte(script), 0o755); err != nil { //nolint:gosec // G306: a hook must be executable to run at all + t.Fatalf("plant on_update: %v", err) + } + + stale := bdCreate(t, bd, dir, "Stale lease", "--type", "task") + live := bdCreate(t, bd, dir, "Live lease", "--type", "task") + bdUpdate(t, bd, dir, stale.ID, "--claim") + bdUpdate(t, bd, dir, live.ID, "--claim") + + cfg, _ := configfile.Load(beadsDir) + database := "" + if cfg != nil { + database = cfg.GetDoltDatabase() + } + dataDir := filepath.Join(beadsDir, "embeddeddolt") + withDB := func(fn func(exec func(string, ...any))) { + t.Helper() + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), dataDir, database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + fn(func(query string, args ...any) { + t.Helper() + if _, err := db.ExecContext(t.Context(), query, args...); err != nil { + t.Fatalf("%s: %v", query, err) + } + }) + } + countCommits := func() int { + t.Helper() + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), dataDir, database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + var n int + if err := db.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM dolt_log WHERE message LIKE 'bd: reclaim%'").Scan(&n); err != nil { + t.Fatalf("count reclaim commits: %v", err) + } + return n + } + withDB(func(exec func(string, ...any)) { + exec("UPDATE leases SET lease_expires_at = ? WHERE issue_id = ?", time.Now().UTC().Add(-time.Hour), stale.ID) + }) + before := countCommits() + + out, err := bdRunWithFlockRetry(t, bd, dir, "reclaim", "--older-than", "0s", "--json", + "--id", stale.ID, "--id", live.ID, "--id", "rc-absent") + if err != nil { + t.Fatalf("bd reclaim: %v\n%s", err, out) + } + var got struct { + Count int `json:"count"` + Scoped bool `json:"scoped"` + Reclaimed []types.ReclaimedLease `json:"reclaimed"` + } + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("unmarshal: %v\n%s", err, out) + } + if got.Count != 1 || len(got.Reclaimed) != 1 || got.Reclaimed[0].ID != stale.ID || !got.Scoped { + t.Fatalf("reclaim answered %+v, want exactly %s, scoped", got, stale.ID) + } + if _, err := types.ParseRevisionToken(got.Reclaimed[0].Revision); err != nil || got.Reclaimed[0].Revision == "" { + t.Fatalf("reclaimed revision %q is not a revision token: %v", got.Reclaimed[0].Revision, err) + } + + if after := countCommits(); after != before+1 { + t.Fatalf("reclaim version commits %d -> %d, want exactly one more", before, after) + } + if iss := bdShow(t, bd, dir, stale.ID); iss.Status != types.StatusOpen || iss.Assignee != "" { + t.Fatalf("%s after reclaim: status=%s assignee=%q, want open and unassigned", stale.ID, iss.Status, iss.Assignee) + } + if iss := bdShow(t, bd, dir, live.ID); iss.Status != types.StatusInProgress { + t.Fatalf("%s after reclaim: status=%s, want it still in_progress", live.ID, iss.Status) + } + + deadline := time.Now().Add(15 * time.Second) + for { + data, _ := os.ReadFile(hookLog) // #nosec G304 -- this test's own temp file + fired := map[string]int{} + for _, line := range strings.Fields(string(data)) { + fired[line]++ + } + // The claims fired on_update for both ids already; the reclaim adds + // one more for the reverted id and none for the live one. + if fired[stale.ID] == 2 && fired[live.ID] == 1 { + break + } + if time.Now().After(deadline) { + t.Fatalf("on_update fired %v, want %s twice (claim, reclaim) and %s once (claim)", fired, stale.ID, live.ID) + } + time.Sleep(50 * time.Millisecond) + } +} diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index fd5b1ad76..4a1592ccf 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -318,6 +318,7 @@ func runServe() error { EdgeReader: roles.edges, GraphCounter: roles.edgeCounter, BatchGetter: roles.batchGetter, + LeaseReclaimer: roles.leaseReclaimer, Relations: roles.relations, Commenter: roles.commenter, BlockingAnnotator: roles.blocking, @@ -674,6 +675,7 @@ type serveRoleSource interface { EdgeReader() (issueops.EdgeReader, error) GraphCounter() (issueops.GraphCounter, error) BatchGetter() (issueops.BatchGetter, error) + LeaseReclaimer() (issueops.LeaseReclaimer, error) IssueRelations() (issueops.Relations, error) Commenter() (issueops.Commenter, error) BlockingAnnotator() (issueops.BlockingAnnotator, error) @@ -741,6 +743,7 @@ func serveIssueRoles(src serveRoleSource, journalEnabled bool) (serveRoles, erro {"edge reader", func() (err error) { roles.edges, err = src.EdgeReader(); return }}, {"graph counter", func() (err error) { roles.edgeCounter, err = src.GraphCounter(); return }}, {"batch getter", func() (err error) { roles.batchGetter, err = src.BatchGetter(); return }}, + {"lease reclaimer", func() (err error) { roles.leaseReclaimer, err = src.LeaseReclaimer(); return }}, {"issue relations", func() (err error) { roles.relations, err = src.IssueRelations(); return }}, {"commenter", func() (err error) { roles.commenter, err = src.Commenter(); return }}, {"blocking annotator", func() (err error) { roles.blocking, err = src.BlockingAnnotator(); return }}, @@ -853,17 +856,22 @@ type serveRoles struct { // getter WITHOUT wrapping it — a read fires no completion hooks — so it is // taken off the peeled store with the rest for uniformity rather than out // of necessity. - batchGetter issueops.BatchGetter - relations issueops.Relations - commenter issueops.Commenter - blocking issueops.BlockingAnnotator - tree issueops.TreeWalker - readyCounter issueops.ReadyCounter - counter issueops.Counter - querier issueops.Querier - sweeper issueops.Sweeper - deleter issueops.Deleter - batchCreator issueops.BatchCreator + batchGetter issueops.BatchGetter + // leaseReclaimer is the stale-lease sweep behind + // POST /v0/beads/issues:reclaim. Its hook decorator WRAPS (a reclaim is an + // update per reverted row), which is why it comes off the peeled store with + // the releaser: bd serve runs no hooks. + leaseReclaimer issueops.LeaseReclaimer + relations issueops.Relations + commenter issueops.Commenter + blocking issueops.BlockingAnnotator + tree issueops.TreeWalker + readyCounter issueops.ReadyCounter + counter issueops.Counter + querier issueops.Querier + sweeper issueops.Sweeper + deleter issueops.Deleter + batchCreator issueops.BatchCreator // dependencyEditor is the second role here whose accessor recurses through // the hook decorator, so taking it off the peeled store is not optional: // HookFiringStore.DependencyEditor fires the workspace's update hook per diff --git a/cmd/bd/serve_source_test.go b/cmd/bd/serve_source_test.go index a0abf2235..1cb7ac105 100644 --- a/cmd/bd/serve_source_test.go +++ b/cmd/bd/serve_source_test.go @@ -100,6 +100,7 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { batchCloser: &serveStubBatchCloser{}, readyClaimer: &serveStubReadyClaimer{}, releaser: &serveStubReleaser{}, + reclaimer: &serveStubLeaseReclaimer{}, lifecycle: &serveStubLifecycle{}, dependencies: &serveStubDependencyEditor{}, batchApplier: &serveStubBatchApplier{}, @@ -187,6 +188,15 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { t.Fatal("the store's own accessor no longer returns a hook-firing releaser; this test proves nothing") } + // And for the lease reclaimer, which fires once per reverted row. + reclaimerFromTheStore, err := chained.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer: %v", err) + } + if !storage.RoleFiresHooks(reclaimerFromTheStore) { + t.Fatal("the store's own accessor no longer returns a hook-firing lease reclaimer; this test proves nothing") + } + // And for the commenter, the SEVENTH — and the one that was OUTSIDE the // RoleFiresHooks switch entirely until the add-comment operation went on the // wire. hook_commenter.go has fired the update hook for every comment it @@ -254,6 +264,9 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { if storage.RoleFiresHooks(roles.releaser) { t.Error("bd serve would run this workspace's hooks on every HTTP release") } + if storage.RoleFiresHooks(roles.leaseReclaimer) || roles.leaseReclaimer != issueops.LeaseReclaimer(middle.reclaimer) { + t.Errorf("lease reclaimer came from %p, want the layer directly beneath the hooks (%p)", roles.leaseReclaimer, middle.reclaimer) + } reader, claimer := roles.reader, roles.claimer // The same predicate httpapi.Listen refuses on, so a regression here is a // server that refuses to boot rather than one that runs hooks silently. @@ -406,6 +419,7 @@ type serveRolesStore struct { batchCloser *serveStubBatchCloser readyClaimer *serveStubReadyClaimer releaser *serveStubReleaser + reclaimer *serveStubLeaseReclaimer lifecycle *serveStubLifecycle dependencies *serveStubDependencyEditor batchApplier *serveStubBatchApplier @@ -459,6 +473,13 @@ func (s *serveRolesStore) ReadyClaimer() (issueops.ReadyClaimer, error) { return // reaper draining abandoned work does in a tight loop. func (s *serveRolesStore) Releaser() (issueops.Releaser, error) { return s.releaser, nil } +// LeaseReclaimer carries an identifiable value for the releaser's reason, and +// for a sharper form of it: its decorator fires on_update once per reverted +// row, so an unpeeled reclaimer runs the workspace's hooks N times per sweep. +func (s *serveRolesStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return s.reclaimer, nil +} + // IssueLifecycle carries an identifiable value for the same reason the reader // and the claimer do: it is one of the roles the hook decorator wraps, so a peel // of the wrong depth would hand bd serve a lifecycle that runs the workspace's @@ -655,6 +676,12 @@ func (*serveStubReleaser) Release(context.Context, issueops.ReleaseRequest) (iss return issueops.ReleaseResult{}, errors.ErrUnsupported } +type serveStubLeaseReclaimer struct{} + +func (*serveStubLeaseReclaimer) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + return issueops.ReclaimResult{}, errors.ErrUnsupported +} + type serveStubLifecycle struct{} func (*serveStubLifecycle) Create(context.Context, issueops.CreateRequest) (issueops.CreateResult, error) { diff --git a/cmd/bd/serve_store_identity_test.go b/cmd/bd/serve_store_identity_test.go index 322adc4c3..4d3efc3f9 100644 --- a/cmd/bd/serve_store_identity_test.go +++ b/cmd/bd/serve_store_identity_test.go @@ -210,6 +210,13 @@ func (*serveIdentityStore) ReadyClaimer() (issueops.ReadyClaimer, error) { return serveIdentityRole{}, nil } func (*serveIdentityStore) Releaser() (issueops.Releaser, error) { return serveIdentityRole{}, nil } + +// LeaseReclaimer is declared at depth 1 for Releaser's reason: its hook +// decorator wraps, and the recursion would otherwise land on a nil-embedded +// promotion. +func (*serveIdentityStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return serveIdentityRole{}, nil +} func (*serveIdentityStore) IssueLifecycle() (issueops.Lifecycle, error) { return serveIdentityRole{}, nil } @@ -297,6 +304,7 @@ func (*serveIdentityStore) MetadataCAS() (issueops.MetadataCAS, error) { type serveIdentityRole struct { issueops.Lifecycle issueops.Releaser + issueops.LeaseReclaimer issueops.ReadyClaimer issueops.BatchCloser issueops.MetadataCAS diff --git a/cmd/bd/unclaim_proxied_integration_test.go b/cmd/bd/unclaim_proxied_integration_test.go index f7fcdfa06..7a64d548e 100644 --- a/cmd/bd/unclaim_proxied_integration_test.go +++ b/cmd/bd/unclaim_proxied_integration_test.go @@ -174,6 +174,7 @@ func TestProxiedServerReclaim(t *testing.T) { Reclaimed []struct { ID string `json:"id"` PreviousOwner string `json:"previous_owner"` + Revision string `json:"revision"` } `json:"reclaimed"` } if err := json.Unmarshal(out, &got); err != nil { @@ -186,6 +187,11 @@ func TestProxiedServerReclaim(t *testing.T) { if r.PreviousOwner == "" { t.Errorf("expected a previous owner for reclaimed %s", issue.ID) } + // The role mints a new revision per reverted row and reports + // it, on this route exactly as on the direct one. + if _, perr := types.ParseRevisionToken(r.Revision); perr != nil || r.Revision == "" { + t.Errorf("reclaimed %s carries revision %q, want a revision token: %v", issue.ID, r.Revision, perr) + } } } if !found { diff --git a/cmd/bd/unclaim_proxied_server.go b/cmd/bd/unclaim_proxied_server.go index 9269beb88..237d4e099 100644 --- a/cmd/bd/unclaim_proxied_server.go +++ b/cmd/bd/unclaim_proxied_server.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "strings" - "time" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/uow" @@ -122,33 +121,3 @@ func runUnclaimProxiedServer(ctx context.Context, args []string, reason string, } return nil } - -func runReclaimProxiedServer(ctx context.Context, olderThan time.Duration, filter types.ReclaimFilter) error { - if uowProvider == nil { - return HandleError("proxied-server UOW provider not initialized") - } - - reclaimed, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) ([]types.ReclaimedLease, string, error) { - out, rerr := uw.IssueUseCase().ReclaimExpiredLeases(ctx, olderThan, filter, currentActor()) - if rerr != nil { - return nil, "", rerr - } - if len(out) == 0 { - return out, "", nil - } - ids := make([]string, 0, len(out)) - for _, r := range out { - ids = append(ids, r.ID) - } - return out, "bd: reclaim " + strings.Join(ids, ", "), nil - }) - if err != nil { - return HandleErrorRespectJSON("reclaim: %v", err) - } - - if len(reclaimed) > 0 { - commandDidWrite.Store(true) - } - - return renderReclaim(reclaimed, !filter.IsEmpty()) -} diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index fb7b25625..27efb854f 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -29,6 +29,7 @@ go_library( "problem.go", "query.go", "reads.go", + "reclaim.go", "related.go", "release.go", "reopen.go", @@ -93,6 +94,7 @@ go_test( "project_stamp_test.go", "reads_test.go", "ready_count_test.go", + "reclaim_test.go", "related_test.go", "release_test.go", "reopen_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 37e13b5f9..4fea37994 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1047,7 +1047,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -1733,6 +1733,44 @@ type ReadyPage struct { Items []IssueWithCounts `json:"items"` } +// ReclaimIssuesRequest One sweep of stale leases. Every member but `actor` is optional, and every scope member NARROWS the sweep; see the operation. +// +// `additionalProperties: false`, so an unknown member is a `400` naming the member — a narrowing term this server silently ignored would widen what is reverted. +type ReclaimIssuesRequest struct { + // Actor Who ran the sweep. REQUIRED: each reverted row's recovery event and history entry are attributed to it. Validated by the same rules as the claim's `actor`. + Actor string `json:"actor"` + + // AnyReplica Also revert leases another replica granted. The one member that WIDENS the sweep; see the operation. Absent means `false`. + AnyReplica *bool `json:"any_replica,omitempty"` + + // Assignees Only leases held by one of these owners are eligible. + Assignees *[]string `json:"assignees,omitempty"` + + // ExcludeLabels An issue carrying ANY of these labels is never eligible. + ExcludeLabels *[]string `json:"exclude_labels,omitempty"` + + // Ids Only these issues are eligible. An id that is not currently a stale lease is absent from the answer rather than refused. The cap is on the REQUEST, counted before deduplication. A blank entry is refused. + Ids *[]string `json:"ids,omitempty"` + + // Labels Only issues carrying ALL of these labels are eligible. + Labels *[]string `json:"labels,omitempty"` + + // LabelsAny Only issues carrying AT LEAST ONE of these labels are eligible. + LabelsAny *[]string `json:"labels_any,omitempty"` + + // OlderThanSeconds The grace window past a lease's own expiry, in seconds (fractions allowed): only a lease that expired more than this long ago is eligible. Absent means `0`, every currently expired lease. A negative value is refused. + OlderThanSeconds *float64 `json:"older_than_seconds,omitempty"` +} + +// ReclaimIssuesResult Every row the sweep reverted, in the order the sweep found them. +type ReclaimIssuesResult struct { + // Reclaimed One entry per reverted row. Empty array (never null) when nothing was stale in scope. + Reclaimed []ReclaimedLease `json:"reclaimed"` +} + +// ReclaimedLease One row a lease sweep reverted. +type ReclaimedLease = types.ReclaimedLease + // Ref Names ONE issue, either by an id that already exists or by the `key` a create item earlier in the same request gave itself. // // EXACTLY ONE OF THE TWO IS SET, and both cases the schema cannot express are a `400`: both members set is a caller that cannot say which it meant, and neither set is a reference to nothing. (Spelling that as a schema alternation would need `oneOf`, which this document does not use — see `ApplyItem`.) @@ -2793,6 +2831,9 @@ type ClaimNextIssueJSONRequestBody = ClaimNextRequest // DeleteIssuesJSONRequestBody defines body for DeleteIssues for application/json ContentType. type DeleteIssuesJSONRequestBody = DeleteIssuesRequest +// ReclaimIssuesJSONRequestBody defines body for ReclaimIssues for application/json ContentType. +type ReclaimIssuesJSONRequestBody = ReclaimIssuesRequest + // SweepIssuesJSONRequestBody defines body for SweepIssues for application/json ContentType. type SweepIssuesJSONRequestBody = SweepRequest diff --git a/internal/httpapi/claim.go b/internal/httpapi/claim.go index 24d79093b..5b5db6326 100644 --- a/internal/httpapi/claim.go +++ b/internal/httpapi/claim.go @@ -350,6 +350,7 @@ var ( _ uow.EdgeReaderSource = timedProvider{} _ uow.GraphCounterSource = timedProvider{} _ uow.BatchGetterSource = timedProvider{} + _ uow.LeaseReclaimerSource = timedProvider{} _ uow.RelationsSource = timedProvider{} _ uow.CommenterSource = timedProvider{} _ uow.BlockingAnnotatorSource = timedProvider{} @@ -474,6 +475,12 @@ func (p timedProvider) BatchGetter() (issueops.BatchGetter, error) { return uow.NewBatchGetter(p) } +// LeaseReclaimer builds the stale-lease sweep OVER THIS WRAPPER, for the same +// reason and with the same hazard as IssueReader. +func (p timedProvider) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return uow.NewLeaseReclaimer(p) +} + // IssueRelations builds the single-anchor neighbor role OVER THIS WRAPPER, for // the same reason and with the same hazard as IssueReader. It is the one // accessor here whose name is not the role's: the seam spells it IssueRelations diff --git a/internal/httpapi/pinning.go b/internal/httpapi/pinning.go index 2144038fb..01435a28b 100644 --- a/internal/httpapi/pinning.go +++ b/internal/httpapi/pinning.go @@ -28,6 +28,7 @@ var ( _ apigen.IssueWithCounts = types.IssueWithCounts{} _ apigen.IssueDetails = types.IssueDetails{} _ apigen.BatchGetIssue = types.BatchGetIssue{} + _ apigen.ReclaimedLease = types.ReclaimedLease{} _ apigen.IssueWithDependencyMetadata = types.IssueWithDependencyMetadata{} _ apigen.Dependency = types.Dependency{} _ apigen.TreeNode = types.TreeNode{} @@ -73,6 +74,7 @@ var ( _ []issueops.IssueBlocking = apigen.BlockingAnnotations{}.Items _ []types.Issue = apigen.BatchCreateResponse{}.Items _ []types.BatchGetIssue = apigen.BatchGetIssuesResult{}.Issues + _ []types.ReclaimedLease = apigen.ReclaimIssuesResult{}.Reclaimed _ []eventsjournal.Record = apigen.EventsPage{}.Records ) diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index d3252ed9a..87618d3bd 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -466,6 +466,12 @@ const ( // answer EdgeCountRequest.IDs and EdgeReadRequest already give, applied to // the issues themselves rather than their edges. OpBatchGetIssues = "batchGetIssues" + // OpReclaimIssues reverts every stale lease a request scopes back to ready, + // behind issueops.LeaseReclaimer. It is NOT releaseIssue repeated: that + // operation gives up ONE claim on the holder's say-so and refuses an unheld + // row, and this one sweeps MANY leases on a clock's say-so and reports an id + // that was not a stale lease by leaving it out of `reclaimed`. + OpReclaimIssues = "reclaimIssues" // OpListRelatedIssues reads ONE issue's neighbors in a named direction, // behind issueops.Relations. It is NOT listDependencies narrowed to one // anchor: that operation answers the stored edge ROWS with their targets @@ -720,6 +726,11 @@ var operationCodes = map[string][]Code{ // `ids` (*issueops.TooManyIDsError) and a blank entry. An empty `ids` is // legal and answers with an empty result, so it earns no refusal at all. OpBatchGetIssues: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, + // No not_found and no conflict: an id that is not a stale lease is left out + // of `reclaimed` rather than refused, and the sweep guards nothing a caller + // named a version of. Its only 400s are the role's own refusals and the + // handler's shape checks. + OpReclaimIssues: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, // The same vocabulary as the stored-edge read beside it, and no not_found // for a stronger version of the same reason: this operation probes no id's // existence at all, so there is nothing it could 404 on. diff --git a/internal/httpapi/reclaim.go b/internal/httpapi/reclaim.go new file mode 100644 index 000000000..62f55a108 --- /dev/null +++ b/internal/httpapi/reclaim.go @@ -0,0 +1,249 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "math" + "net/http" + "slices" + "strings" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// The request body's member vocabulary. The schema is +// additionalProperties: false, so anything else is refused BY NAME, and on this +// operation for the sweep's sharper reason: a scope term the server silently +// ignored would widen what is reverted. +const ( + reclaimActorMember = "actor" + reclaimOlderThanMember = "older_than_seconds" + reclaimIDsMember = "ids" + reclaimAssigneesMember = "assignees" + reclaimLabelsMember = "labels" + reclaimLabelsAnyMember = "labels_any" + reclaimExcludeLabelsMember = "exclude_labels" + reclaimAnyReplicaMember = "any_replica" +) + +// reclaimMembers is the whole vocabulary, in one place, so the unknown-member +// refusal and the decoding below cannot come to disagree about what this +// operation accepts. +var reclaimMembers = []string{ + reclaimActorMember, + reclaimOlderThanMember, + reclaimIDsMember, + reclaimAssigneesMember, + reclaimLabelsMember, + reclaimLabelsAnyMember, + reclaimExcludeLabelsMember, + reclaimAnyReplicaMember, +} + +// maxReclaimOlderThanSeconds is the smallest grace window time.Duration can +// NOT hold: MaxInt64/1e9 as a float64 rounds up, so converting it back yields +// 2^63 nanoseconds, which wraps negative. The check is >=, so every accepted +// value converts in range; a refused one is not wrapped into a negative +// duration the role would refuse with a message about a value never sent. +var maxReclaimOlderThanSeconds = float64(math.MaxInt64) / float64(time.Second) + +// handleReclaimIssues answers POST /v0/beads/issues:reclaim. +// +// WHAT THIS HANDLER DOES NOT DO: it does not decide which lease is stale, does +// not apply the scope, does not honor the replica guard and does not mint the +// revisions. All of that is issueops.LeaseReclaimer, the same library surface +// `bd reclaim` calls, so a second front door cannot sweep differently from the +// first. Everything above the role here is argument validation and projection. +// +// THE ACTOR IS REQUIRED and validated by the claim's rules: every reverted row +// records a recovery event attributed to it, and the server's own identity +// would be meaningless to a remote reaper. +func (s *Server) handleReclaimIssues(w http.ResponseWriter, r *http.Request) { + if !s.requireNoQuery(w, r) { + return + } + if !s.requireJSONContent(w, r) { + return + } + request, ok := s.reclaimRequest(w, r) + if !ok { + return + } + + reclaimer, err := s.leaseReclaimer(r) + if err != nil { + s.failErr(w, r, err) + return + } + result, err := reclaimer.Reclaim(r.Context(), request) + if err != nil { + s.failReclaimErr(w, r, err) + return + } + writeJSON(w, reclaimResponse(result)) +} + +// reclaimRequest decodes the body into the role's request, member by member, +// so every refusal can NAME the member it is about. +func (s *Server) reclaimRequest(w http.ResponseWriter, r *http.Request) (issueops.ReclaimRequest, bool) { + members, res := decodeJSONObjectBody(w, r) + if res != nil { + s.fail(w, r, *res) + return issueops.ReclaimRequest{}, false + } + + var unknown []string + for name := range members { + if !slices.Contains(reclaimMembers, name) { + unknown = append(unknown, name) + } + } + if len(unknown) > 0 { + offender := slices.Min(unknown) + requestInfo(r.Context()).refuse(offender) + s.fail(w, r, InvalidArgument(offender, ReasonUnknownParameter, + "this operation's request body carries "+reclaimMemberList()+" and nothing else")) + return issueops.ReclaimRequest{}, false + } + + var request issueops.ReclaimRequest + + raw, ok := members[reclaimActorMember] + if !ok { + s.fail(w, r, InvalidArgument(reclaimActorMember, ReasonInvalidValue, + "`"+reclaimActorMember+"` is required")) + return issueops.ReclaimRequest{}, false + } + var actor *string + if err := json.Unmarshal(raw, &actor); err != nil || actor == nil { + s.fail(w, r, InvalidArgument(reclaimActorMember, ReasonInvalidValue, + "`"+reclaimActorMember+"` must be a string")) + return issueops.ReclaimRequest{}, false + } + trimmed, res := validateActor(*actor) + if res != nil { + s.fail(w, r, *res) + return issueops.ReclaimRequest{}, false + } + request.Actor = trimmed + + if raw, ok := members[reclaimOlderThanMember]; ok { + var seconds *float64 + if err := json.Unmarshal(raw, &seconds); err != nil || seconds == nil { + s.fail(w, r, InvalidArgument(reclaimOlderThanMember, ReasonInvalidValue, + "`"+reclaimOlderThanMember+"` must be a number")) + return issueops.ReclaimRequest{}, false + } + if *seconds < 0 || *seconds >= maxReclaimOlderThanSeconds { + s.fail(w, r, InvalidArgument(reclaimOlderThanMember, ReasonInvalidValue, + "`"+reclaimOlderThanMember+"` must be a non-negative number of seconds a duration can hold")) + return issueops.ReclaimRequest{}, false + } + request.OlderThan = time.Duration(*seconds * float64(time.Second)) + } + + for _, scope := range []struct { + member string + dest *[]string + }{ + {reclaimIDsMember, &request.Filter.IDs}, + {reclaimAssigneesMember, &request.Filter.Assignees}, + {reclaimLabelsMember, &request.Filter.Labels}, + {reclaimLabelsAnyMember, &request.Filter.LabelsAny}, + {reclaimExcludeLabelsMember, &request.Filter.ExcludeLabels}, + } { + raw, ok := members[scope.member] + if !ok { + continue + } + var values *[]string + if err := json.Unmarshal(raw, &values); err != nil || values == nil { + s.fail(w, r, InvalidArgument(scope.member, ReasonInvalidValue, + "`"+scope.member+"` must be an array of strings")) + return issueops.ReclaimRequest{}, false + } + // A PRESENT, EMPTY scope is refused rather than read as "no scope". + // The two would sweep the same rows, and that is the hazard: `bd + // reclaim --label "$LANE"` with LANE unset refuses for exactly this + // reason, so a caller that meant to narrow and sent nothing gets told + // rather than getting a workspace-wide sweep. + if len(*values) == 0 { + s.fail(w, r, InvalidArgument(scope.member, ReasonInvalidValue, + "`"+scope.member+"` is empty; an empty scope would reclaim everything, so omit the member to sweep without it")) + return issueops.ReclaimRequest{}, false + } + // The id cap and every blank entry are the ROLE's refusals + // (ValidateReclaimRequest, the one definition the CLI shares) and + // reach the wire through failReclaimErr, which names the field the + // role names. + *scope.dest = *values + } + + if raw, ok := members[reclaimAnyReplicaMember]; ok { + var value *bool + if err := json.Unmarshal(raw, &value); err != nil || value == nil { + s.fail(w, r, InvalidArgument(reclaimAnyReplicaMember, ReasonInvalidValue, + "`"+reclaimAnyReplicaMember+"` must be a boolean")) + return issueops.ReclaimRequest{}, false + } + request.Filter.AnyReplica = *value + } + + return request, true +} + +func reclaimMemberList() string { + quoted := make([]string, len(reclaimMembers)) + for i, name := range reclaimMembers { + quoted[i] = "`" + name + "`" + } + return strings.Join(quoted, ", ") +} + +// reclaimFieldMembers spells each field the role's *ReclaimFieldError can +// name as the wire member it arrived in. +var reclaimFieldMembers = map[string]string{ + issueops.ReclaimFieldActor: reclaimActorMember, + issueops.ReclaimFieldOlderThan: reclaimOlderThanMember, + issueops.ReclaimFieldIDs: reclaimIDsMember, + issueops.ReclaimFieldAssignees: reclaimAssigneesMember, + issueops.ReclaimFieldLabels: reclaimLabelsMember, + issueops.ReclaimFieldLabelsAny: reclaimLabelsAnyMember, + issueops.ReclaimFieldExcludeLabels: reclaimExcludeLabelsMember, +} + +// failReclaimErr answers a failed reclaim. issueops.ErrValidation is mapped to +// a 400 HERE, in the sweep's shape, because the role validates what the +// handler does not duplicate. The refusal names the member the role's error +// is about: `ids` for the cap, the *ReclaimFieldError's own field otherwise, +// and no member at all for a validation that names none. +func (s *Server) failReclaimErr(w http.ResponseWriter, r *http.Request, err error) { + if !errors.Is(err, issueops.ErrValidation) { + s.failErr(w, r, err) + return + } + member := "" + var capErr *issueops.TooManyReclaimIDsError + var fieldErr *issueops.ReclaimFieldError + switch { + case errors.As(err, &capErr): + member = reclaimIDsMember + case errors.As(err, &fieldErr): + member = reclaimFieldMembers[fieldErr.Field] + } + if member != "" { + requestInfo(r.Context()).refuse(member) + } + s.fail(w, r, InvalidArgument(member, ReasonInvalidValue, err.Error())) +} + +// reclaimResponse projects the role's result onto the wire envelope. +// apigen.ReclaimedLease IS types.ReclaimedLease (pinning.go), so this is a +// copy rather than a field list; `reclaimed` is never null. +func reclaimResponse(result issueops.ReclaimResult) apigen.ReclaimIssuesResult { + body := apigen.ReclaimIssuesResult{Reclaimed: []apigen.ReclaimedLease{}} + body.Reclaimed = append(body.Reclaimed, result.Reclaimed...) + return body +} diff --git a/internal/httpapi/reclaim_test.go b/internal/httpapi/reclaim_test.go new file mode 100644 index 000000000..4db65aeda --- /dev/null +++ b/internal/httpapi/reclaim_test.go @@ -0,0 +1,317 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// These cover the transport half of POST /v0/beads/issues:reclaim: the body +// shape, the refusals the handler raises itself, how the role's own refusals +// reach the wire, and the projection of the role's result. Everything below +// the wire (staleness, scope, the replica guard, the minted revision, the hook) +// is issueops.LeaseReclaimer's, pinned by +// backend/conformance/lease_reclaimer_contract.go on all four legs. + +const reclaimPath = "/v0/beads/issues:reclaim" + +func newReclaimServer(t *testing.T, reclaimer *roleLeaseReclaimer) *testServer { + t.Helper() + return newTestServer(t, rolesConfig(Config{LeaseReclaimer: reclaimer})) +} + +// TestReclaimIssuesMapsEveryMemberOntoTheRole pins the request half: every +// member reaches the role's request, the actor trimmed by the claim's rules and +// the fractional grace window converted exactly. +func TestReclaimIssuesMapsEveryMemberOntoTheRole(t *testing.T) { + reclaimer := &roleLeaseReclaimer{result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{}}} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, `{ + "actor": " reaper ", + "older_than_seconds": 90.5, + "ids": ["bd-1", "bd-2"], + "assignees": ["w1"], + "labels": ["lane-a"], + "labels_any": ["x", "y"], + "exclude_labels": ["pinned"], + "any_replica": true + }`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + calls := reclaimer.reclaimRequests() + if len(calls) != 1 { + t.Fatalf("role calls = %d, want 1", len(calls)) + } + want := issueops.ReclaimRequest{ + Actor: "reaper", + OlderThan: 90*time.Second + 500*time.Millisecond, + Filter: issueops.ReclaimFilter{ + IDs: []string{"bd-1", "bd-2"}, + Assignees: []string{"w1"}, + Labels: []string{"lane-a"}, + LabelsAny: []string{"x", "y"}, + ExcludeLabels: []string{"pinned"}, + AnyReplica: true, + }, + } + if !reflect.DeepEqual(calls[0], want) { + t.Fatalf("role request = %+v, want %+v", calls[0], want) + } +} + +// TestReclaimIssuesDefaultsToAnUnscopedZeroGraceSweep pins the absent members: +// no grace window is zero, no scope is no scope, and any_replica is off. +func TestReclaimIssuesDefaultsToAnUnscopedZeroGraceSweep(t *testing.T) { + reclaimer := &roleLeaseReclaimer{result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{}}} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, `{"actor":"reaper"}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + calls := reclaimer.reclaimRequests() + if len(calls) != 1 || !reflect.DeepEqual(calls[0], issueops.ReclaimRequest{Actor: "reaper"}) { + t.Fatalf("role calls = %+v, want one bare request from reaper", calls) + } +} + +// TestReclaimIssuesAnswersEveryReclaimedLeaseWithItsRevision pins the response +// half: each entry carries id, previous_owner and the role's revision as the +// decimal string, past 2^53 so a float64 anywhere on the path would show. +func TestReclaimIssuesAnswersEveryReclaimedLeaseWithItsRevision(t *testing.T) { + reclaimer := &roleLeaseReclaimer{result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{ + {ID: "bd-1", PreviousOwner: "w1", Revision: types.RevisionToken(guardToken)}, + {ID: "bd-2", PreviousOwner: "", Revision: "7"}, + }}} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, `{"actor":"reaper"}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + raw := readAll(t, resp) + var body struct { + Reclaimed []map[string]any `json:"reclaimed"` + } + if err := json.Unmarshal([]byte(raw), &body); err != nil { + t.Fatalf("decode %q: %v", raw, err) + } + if len(body.Reclaimed) != 2 { + t.Fatalf("reclaimed = %v, want 2 entries", body.Reclaimed) + } + first := body.Reclaimed[0] + if first["id"] != "bd-1" || first["previous_owner"] != "w1" || first["revision"] != types.RevisionToken(guardToken) { + t.Errorf("reclaimed[0] = %v, want {bd-1 w1 %s}", first, types.RevisionToken(guardToken)) + } + if second := body.Reclaimed[1]; second["previous_owner"] != "" || second["revision"] != "7" { + t.Errorf("reclaimed[1] = %v, want an empty previous_owner and revision \"7\"", second) + } +} + +// TestReclaimIssuesAnswersAnEmptyArrayNotNull pins `reclaimed: []` when the +// sweep found nothing, whatever the role handed back. +func TestReclaimIssuesAnswersAnEmptyArrayNotNull(t *testing.T) { + for _, result := range []issueops.ReclaimResult{{}, {Reclaimed: []issueops.ReclaimedLease{}}} { + ts := newReclaimServer(t, &roleLeaseReclaimer{result: result}) + resp := ts.claim(t, reclaimPath, `{"actor":"reaper","ids":["bd-gone"]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + if raw := readAll(t, resp); !strings.Contains(raw, `"reclaimed":[]`) { + t.Errorf("body = %s, want `\"reclaimed\":[]`", raw) + } + } +} + +// TestReclaimIssuesRefusesAMalformedBodyByName pins every refusal the handler +// raises itself: each is a 400 naming its member, and none reaches the role. +func TestReclaimIssuesRefusesAMalformedBodyByName(t *testing.T) { + for _, tc := range []struct { + name, body, param, reason string + }{ + {"unknown member", `{"actor":"r","force":true}`, "force", string(ReasonUnknownParameter)}, + {"missing actor", `{"ids":["bd-1"]}`, "actor", string(ReasonInvalidValue)}, + {"null actor", `{"actor":null}`, "actor", string(ReasonInvalidValue)}, + {"blank actor", `{"actor":" "}`, "actor", ""}, + {"negative grace", `{"actor":"r","older_than_seconds":-1}`, "older_than_seconds", string(ReasonInvalidValue)}, + {"string grace", `{"actor":"r","older_than_seconds":"10m"}`, "older_than_seconds", string(ReasonInvalidValue)}, + {"grace past a duration", `{"actor":"r","older_than_seconds":1e300}`, "older_than_seconds", string(ReasonInvalidValue)}, + {"ids not an array", `{"actor":"r","ids":"bd-1"}`, "ids", string(ReasonInvalidValue)}, + {"empty ids", `{"actor":"r","ids":[]}`, "ids", string(ReasonInvalidValue)}, + {"empty labels", `{"actor":"r","labels":[]}`, "labels", string(ReasonInvalidValue)}, + {"empty assignees", `{"actor":"r","assignees":[]}`, "assignees", string(ReasonInvalidValue)}, + {"null exclude", `{"actor":"r","exclude_labels":null}`, "exclude_labels", string(ReasonInvalidValue)}, + {"any_replica not a bool", `{"actor":"r","any_replica":"yes"}`, "any_replica", string(ReasonInvalidValue)}, + } { + t.Run(tc.name, func(t *testing.T) { + reclaimer := &roleLeaseReclaimer{} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, tc.body) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) { + t.Errorf("code = %v, want %q", body["code"], CodeInvalidArgument) + } + if body["param"] != tc.param { + t.Errorf("param = %v, want %s", body["param"], tc.param) + } + if tc.reason != "" && body["reason"] != tc.reason { + t.Errorf("reason = %v, want %s", body["reason"], tc.reason) + } + if calls := reclaimer.reclaimRequests(); len(calls) != 0 { + t.Errorf("the role was called %d times for a refused request", len(calls)) + } + }) + } +} + +// TestReclaimIssuesRefusesAQueryParameter pins the document-level +// unknown-parameter rule on this operation, which declares none. +func TestReclaimIssuesRefusesAQueryParameter(t *testing.T) { + reclaimer := &roleLeaseReclaimer{} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath+"?older_than=1h", `{"actor":"reaper"}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["param"] != "older_than" || body["reason"] != string(ReasonUnknownParameter) { + t.Errorf("param/reason = %v/%v, want older_than/unknown_parameter", body["param"], body["reason"]) + } + if calls := reclaimer.reclaimRequests(); len(calls) != 0 { + t.Errorf("the role was called %d times for a refused request", len(calls)) + } +} + +// TestReclaimIssuesLeavesTheIDsCapToTheRole pins that the handler neither +// counts nor trims `ids`: the whole oversized request reaches the role, and +// the role's *TooManyReclaimIDsError comes back as a 400 naming `ids`. +func TestReclaimIssuesLeavesTheIDsCapToTheRole(t *testing.T) { + const over = issueops.MaxReclaimIDs + 1 + reclaimer := &roleLeaseReclaimer{err: &issueops.TooManyReclaimIDsError{Requested: over, Cap: issueops.MaxReclaimIDs}} + ts := newReclaimServer(t, reclaimer) + + ids := make([]string, over) + for i := range ids { + ids[i] = fmt.Sprintf(`"bd-%d"`, i) + } + resp := ts.claim(t, reclaimPath, `{"actor":"reaper","ids":[`+strings.Join(ids, ",")+`]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) || body["param"] != "ids" || body["reason"] != string(ReasonInvalidValue) { + t.Errorf("problem = %v, want invalid_argument/ids/invalid_value", body) + } + calls := reclaimer.reclaimRequests() + if len(calls) != 1 || len(calls[0].Filter.IDs) != over { + t.Fatalf("role calls = %d, want one carrying all %d ids", len(calls), over) + } +} + +// TestReclaimIssuesAnswersABlankIDThroughTheRole pins the other role-side +// refusal: a blank entry in `ids` reaches the role (it is not pre-checked +// here), and its ErrValidation is a 400 naming `ids`. +func TestReclaimIssuesAnswersABlankIDThroughTheRole(t *testing.T) { + reclaimer := &roleLeaseReclaimer{err: &issueops.ReclaimFieldError{ + Field: issueops.ReclaimFieldIDs, Detail: "validation failed: reclaim scope id at position 1 is empty", + }} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, `{"actor":"reaper","ids":["bd-1",""]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + if body := decodeBody(t, resp); body["param"] != "ids" { + t.Errorf("param = %v, want ids", body["param"]) + } + if calls := reclaimer.reclaimRequests(); len(calls) != 1 { + t.Fatalf("role calls = %d, want 1", len(calls)) + } +} + +// TestReclaimIssuesNamesTheFieldTheRoleRefused pins that a role refusal is +// attributed to the member the role's *ReclaimFieldError names, not to `ids` +// by default, and that the handler leaves blank scope entries to the role: +// the request reaches it. +func TestReclaimIssuesNamesTheFieldTheRoleRefused(t *testing.T) { + for field, member := range reclaimFieldMembers { + t.Run(field, func(t *testing.T) { + reclaimer := &roleLeaseReclaimer{err: &issueops.ReclaimFieldError{Field: field, Detail: "validation failed: refused"}} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, `{"actor":"reaper","labels_any":["a"," "]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + if body := decodeBody(t, resp); body["param"] != member { + t.Errorf("param = %v, want %s", body["param"], member) + } + calls := reclaimer.reclaimRequests() + if len(calls) != 1 || !reflect.DeepEqual(calls[0].Filter.LabelsAny, []string{"a", " "}) { + t.Fatalf("role calls = %+v, want one carrying the blank entry unfiltered", calls) + } + }) + } +} + +// TestReclaimFieldMembersCoverEveryRoleField pins the attribution table +// against the role's whole field vocabulary, so a field added to the role +// cannot reach the wire with no member named. +func TestReclaimFieldMembersCoverEveryRoleField(t *testing.T) { + for _, field := range []string{ + issueops.ReclaimFieldActor, issueops.ReclaimFieldOlderThan, issueops.ReclaimFieldIDs, + issueops.ReclaimFieldAssignees, issueops.ReclaimFieldLabels, issueops.ReclaimFieldLabelsAny, + issueops.ReclaimFieldExcludeLabels, + } { + if reclaimFieldMembers[field] == "" { + t.Errorf("role field %q has no wire member", field) + } + } +} + +// TestReclaimIssuesRefusesTheFirstUnrepresentableGraceWindow pins the +// boundary: float64(MaxInt64)/1e9 converts back to 2^63 nanoseconds, which +// wraps negative, so that exact value is refused, not passed on. +func TestReclaimIssuesRefusesTheFirstUnrepresentableGraceWindow(t *testing.T) { + reclaimer := &roleLeaseReclaimer{} + ts := newReclaimServer(t, reclaimer) + + value := strconv.FormatFloat(maxReclaimOlderThanSeconds, 'g', -1, 64) + resp := ts.claim(t, reclaimPath, `{"actor":"reaper","older_than_seconds":`+value+`}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("older_than_seconds=%s: status = %d, want 400: %s", value, resp.StatusCode, readAll(t, resp)) + } + if body := decodeBody(t, resp); body["param"] != "older_than_seconds" { + t.Errorf("param = %v, want older_than_seconds", body["param"]) + } + if calls := reclaimer.reclaimRequests(); len(calls) != 0 { + t.Fatalf("the role received %+v; an out-of-range window must not reach it", calls) + } +} + +// TestReclaimIssuesDoesNotDisguiseAStorageFailure pins that a failure that is +// not a validation is not answered as the caller's fault. +func TestReclaimIssuesDoesNotDisguiseAStorageFailure(t *testing.T) { + ts := newReclaimServer(t, &roleLeaseReclaimer{err: errors.New("disk on fire")}) + + resp := ts.claim(t, reclaimPath, `{"actor":"reaper"}`) + if resp.StatusCode < 500 { + t.Fatalf("status = %d, want a 5xx: %s", resp.StatusCode, readAll(t, resp)) + } +} diff --git a/internal/httpapi/roles_test.go b/internal/httpapi/roles_test.go index 083923e9c..6ada7952d 100644 --- a/internal/httpapi/roles_test.go +++ b/internal/httpapi/roles_test.go @@ -162,6 +162,34 @@ func (g *roleBatchGetter) getManyRequests() []issueops.GetManyRequest { return append([]issueops.GetManyRequest(nil), g.calls...) } +// roleLeaseReclaimer is the stale-lease sweep of the store-shaped source, its +// own fake beside roleReleaser for roleBatchGetter's reason: a separate role on +// a separate accessor, so one fake answering both would let a test pass on a +// server that had wired the reclaim handler to the wrong role. +type roleLeaseReclaimer struct { + result issueops.ReclaimResult + err error + + mu sync.Mutex + calls []issueops.ReclaimRequest +} + +func (l *roleLeaseReclaimer) Reclaim(_ context.Context, req issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + l.mu.Lock() + l.calls = append(l.calls, req) + l.mu.Unlock() + if l.err != nil { + return issueops.ReclaimResult{}, l.err + } + return l.result, nil +} + +func (l *roleLeaseReclaimer) reclaimRequests() []issueops.ReclaimRequest { + l.mu.Lock() + defer l.mu.Unlock() + return append([]issueops.ReclaimRequest(nil), l.calls...) +} + // roleRelations is the single-anchor NEIGHBOR role of the store-shaped source, // its own fake beside roleEdgeReader for roleGraphCounter's reason: the two sit // on adjacent accessors and answer about the same edges, so one fake serving @@ -960,6 +988,9 @@ func rolesConfig(cfg Config) Config { if cfg.BatchGetter == nil { cfg.BatchGetter = &roleBatchGetter{} } + if cfg.LeaseReclaimer == nil { + cfg.LeaseReclaimer = &roleLeaseReclaimer{} + } if cfg.Relations == nil { cfg.Relations = &roleRelations{} } diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 4c426275a..1dc78fa94 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -783,6 +783,17 @@ var routeTable = []route{ implemented: true, handler: (*Server).handleBatchGetIssues, }, + { + op: OpReclaimIssues, + method: http.MethodPost, + // A literal collection-level custom method, registered and preferred + // over the claim's wildcard for the sweep row's reason: without it, + // a reclaim would parse as a claim of an issue called ":reclaim". + pattern: "/v0/beads/issues:reclaim", + capability: "issues.reclaim", + implemented: true, + handler: (*Server).handleReclaimIssues, + }, { op: OpAddDependencies, method: http.MethodPost, diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index a3b928390..3a27b520a 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -287,6 +287,12 @@ type Config struct { // rather than refusing the call. Required on the same terms as every field // here. BatchGetter issueops.BatchGetter + // LeaseReclaimer is the stale-lease sweep behind + // POST /v0/beads/issues:reclaim. It is a SEPARATE field from Releaser for + // the reason issueops.LeaseReclaimer's own doc gives: Releaser gives up ONE + // claim on its holder's say-so, and this reverts MANY on a clock's. + // Required on the same terms as every field here. + LeaseReclaimer issueops.LeaseReclaimer // Relations is the single-anchor neighbor read behind // GET /v0/beads/issues/{id}/related. It is a SEPARATE field from EdgeReader // for the reason issueops.EdgeReader's own doc gives at length: that role @@ -423,6 +429,7 @@ type Server struct { issueEdges issueops.EdgeReader issueEdgeCounter issueops.GraphCounter issueBatchGetter issueops.BatchGetter + issueReclaimer issueops.LeaseReclaimer issueRelations issueops.Relations issueCommenter issueops.Commenter issueBlocking issueops.BlockingAnnotator @@ -635,6 +642,7 @@ func Listen(cfg Config) (*Server, error) { issueEdges: cfg.EdgeReader, issueEdgeCounter: cfg.GraphCounter, issueBatchGetter: cfg.BatchGetter, + issueReclaimer: cfg.LeaseReclaimer, issueRelations: cfg.Relations, issueCommenter: cfg.Commenter, issueBlocking: cfg.BlockingAnnotator, @@ -757,12 +765,12 @@ func Listen(cfg Config) (*Server, error) { // "all or nothing" would turn an honest condition into a special case inside // three functions. It is checked once, on its own, below. func sourceRoles(cfg Config) []any { - return []any{cfg.Reader, cfg.Claimer, cfg.ReadyClaimer, cfg.Releaser, cfg.Lifecycle, cfg.BatchCloser, cfg.Settings, cfg.Stats, cfg.CycleDetector, cfg.EdgeReader, cfg.GraphCounter, cfg.BatchGetter, cfg.Relations, cfg.Commenter, cfg.BlockingAnnotator, cfg.TreeWalker, cfg.ReadyCounter, cfg.Counter, cfg.Querier, cfg.Sweeper, cfg.Deleter, cfg.BatchCreator, cfg.DependencyEditor, cfg.BatchApplier, cfg.Memories, cfg.MetadataCAS} + return []any{cfg.Reader, cfg.Claimer, cfg.ReadyClaimer, cfg.Releaser, cfg.Lifecycle, cfg.BatchCloser, cfg.Settings, cfg.Stats, cfg.CycleDetector, cfg.EdgeReader, cfg.GraphCounter, cfg.BatchGetter, cfg.LeaseReclaimer, cfg.Relations, cfg.Commenter, cfg.BlockingAnnotator, cfg.TreeWalker, cfg.ReadyCounter, cfg.Counter, cfg.Querier, cfg.Sweeper, cfg.Deleter, cfg.BatchCreator, cfg.DependencyEditor, cfg.BatchApplier, cfg.Memories, cfg.MetadataCAS} } // roleSourceNames spells sourceRoles for the refusal message, in the same // order, so a caller reading the error learns the whole set it must pass. -const roleSourceNames = "Reader, Claimer, ReadyClaimer, Releaser, Lifecycle, BatchCloser, Settings, Stats, CycleDetector, EdgeReader, GraphCounter, BatchGetter, Relations, Commenter, BlockingAnnotator, TreeWalker, ReadyCounter, Counter, Querier, Sweeper, Deleter, BatchCreator, DependencyEditor, BatchApplier, Memories and MetadataCAS" +const roleSourceNames = "Reader, Claimer, ReadyClaimer, Releaser, Lifecycle, BatchCloser, Settings, Stats, CycleDetector, EdgeReader, GraphCounter, BatchGetter, LeaseReclaimer, Relations, Commenter, BlockingAnnotator, TreeWalker, ReadyCounter, Counter, Querier, Sweeper, Deleter, BatchCreator, DependencyEditor, BatchApplier, Memories and MetadataCAS" func anyRoleSet(cfg Config) bool { return slices.ContainsFunc(sourceRoles(cfg), func(r any) bool { return r != nil }) @@ -1151,6 +1159,19 @@ func (s *Server) batchGetter(r *http.Request) (issueops.BatchGetter, error) { return src.BatchGetter() } +// leaseReclaimer returns the stale-lease sweep surface for one request, on the +// same terms as every role above and held by INTERFACE so +// uow.LeaseReclaimerSource is load-bearing rather than decorative. It goes out +// unwrapped for the sweeper's reason: ReclaimResult is a VALUE whose slice a +// nil-safe range walks. +func (s *Server) leaseReclaimer(r *http.Request) (issueops.LeaseReclaimer, error) { + if s.provider == nil { + return s.issueReclaimer, nil + } + var src uow.LeaseReclaimerSource = timedProvider{inner: s.provider, rec: requestInfo(r.Context())} + return src.LeaseReclaimer() +} + // relations returns the single-anchor neighbor surface for one request, built // the same two ways as edgeReader above and held by INTERFACE so // uow.RelationsSource is load-bearing rather than decorative. diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 456693a41..0c087a175 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -63,7 +63,7 @@ info: outside its own parameter table with `400` / `code: invalid_argument` / `reason: "unknown_parameter"` and `param` naming the offending key. Operations that declare no QUERY parameter at all reject every query key - the same way. Today that is twenty-six of the forty-two operations + the same way. Today that is twenty-seven of the forty-three operations here — `GET /healthz`, `GET /v0/beads/context`, `GET /v0/beads/dependencies/cycles`, `POST /v0/beads/issues`, `PATCH /v0/beads/issues/{id}`, @@ -75,6 +75,7 @@ info: `POST /v0/beads/issues/{id}:casMetadata`, `POST /v0/beads/issues:sweep`, `POST /v0/beads/issues:delete`, `POST /v0/beads/issues:batchGet`, + `POST /v0/beads/issues:reclaim`, `POST /v0/beads/issues:batchCreate`, `POST /v0/beads/issues:batchApply`, `POST /v0/beads/issues:batchClose`, `GET /v0/beads/config`, @@ -3891,6 +3892,95 @@ paths: '503': $ref: '#/components/responses/Unavailable' + /v0/beads/issues:reclaim: + post: + operationId: reclaimIssues + summary: Revert stale leases back to ready + description: >- + Reverts every in-progress issue whose LEASE has gone stale back to + ready: the lease row is deleted, `status` becomes `open`, the assignee + and `started_at` are cleared, and a recovery event naming the former + holder is recorded. It is the operation behind `issueops.LeaseReclaimer` + and `bd reclaim`, and the caller it is for is a CLOCK — a supervisor + reaping the work of workers that died without releasing it — rather + than the worker that held the claim, which releases its own with + `POST /v0/beads/issues/{id}:release`. + + + A lease is stale when its expiry is more than `older_than_seconds` in + the past. A lease a worker is still heartbeating is never eligible, + whatever the request names. + + + ## Scope narrows, never widens + + + Every scope member narrows the sweep and they AND-combine: `ids`, + `assignees`, `labels` (all of them), `labels_any` (at least one) and + `exclude_labels` (none of them). A request with no scope member sweeps + every stale lease this server granted. `any_replica` is the one member + that WIDENS: it disarms the granting-replica guard, so a lease another + replica granted is reverted too. It is an operator escape hatch for a + replica that is gone for good, never a normal setting. + + + AN ID THAT IS NOT A STALE LEASE IS NOT AN ERROR. An id naming no row, or + a row whose lease is live, or a row nobody holds, is simply absent from + `reclaimed`; scoping by id narrows a sweep, it does not assert that + every named id was reclaimable. There is no `404` on this operation. + + + ## A new revision per reverted row + + + Every reverted row's `revision` is minted fresh by the revert, and each + entry of `reclaimed` carries it. A holder that wakes up after its lease + was reclaimed and writes with the revision its claim handed it gets + `412 precondition_failed`, so it cannot land over work another worker + may already have taken. + + + The whole sweep is ONE transaction. A lease a heartbeat rescues between + the snapshot that found it and its row's revert is skipped, not + reported. + + + Available when the server advertises the `issues.reclaim` capability. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ReclaimIssuesRequest' + security: + - bearerToken: [] + responses: + '200': + description: >- + The rows the sweep reverted, each with its former holder and its new + revision. An empty array (never null) when nothing was stale. + content: + application/json: + schema: + $ref: '#/components/schemas/ReclaimIssuesResult' + '400': + description: >- + Invalid request: an unparseable body, an unknown body member, a + missing or invalid `actor`, a negative or non-numeric + `older_than_seconds`, an `ids` carrying more than 1000 entries + (counted before deduplication), or a blank scope entry. + x-bd-codes: [invalid_argument] + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + '401': + $ref: '#/components/responses/Unauthenticated' + '500': + $ref: '#/components/responses/InternalError' + '503': + $ref: '#/components/responses/Unavailable' + /v0/beads/issues:batchCreate: post: operationId: batchCreateIssues @@ -7930,7 +8020,7 @@ components: `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, - `issues.batchCreate`, + `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, @@ -8518,6 +8608,108 @@ components: items: type: string + ReclaimIssuesRequest: + type: object + additionalProperties: false + required: [actor] + description: >- + One sweep of stale leases. Every member but `actor` is optional, and + every scope member NARROWS the sweep; see the operation. + + + `additionalProperties: false`, so an unknown member is a `400` naming + the member — a narrowing term this server silently ignored would widen + what is reverted. + properties: + actor: + type: string + description: >- + Who ran the sweep. REQUIRED: each reverted row's recovery event and + history entry are attributed to it. Validated by the same rules as + the claim's `actor`. + older_than_seconds: + type: number + format: double + minimum: 0 + description: >- + The grace window past a lease's own expiry, in seconds (fractions + allowed): only a lease that expired more than this long ago is + eligible. Absent means `0`, every currently expired lease. A + negative value is refused. + ids: + type: array + maxItems: 1000 + description: >- + Only these issues are eligible. An id that is not currently a stale + lease is absent from the answer rather than refused. The cap is on + the REQUEST, counted before deduplication. A blank entry is + refused. + items: + type: string + assignees: + type: array + description: Only leases held by one of these owners are eligible. + items: + type: string + labels: + type: array + description: Only issues carrying ALL of these labels are eligible. + items: + type: string + labels_any: + type: array + description: Only issues carrying AT LEAST ONE of these labels are eligible. + items: + type: string + exclude_labels: + type: array + description: An issue carrying ANY of these labels is never eligible. + items: + type: string + any_replica: + type: boolean + description: >- + Also revert leases another replica granted. The one member that + WIDENS the sweep; see the operation. Absent means `false`. + + ReclaimIssuesResult: + type: object + required: [reclaimed] + description: >- + Every row the sweep reverted, in the order the sweep found them. + properties: + reclaimed: + type: array + description: >- + One entry per reverted row. Empty array (never null) when nothing + was stale in scope. + items: + $ref: '#/components/schemas/ReclaimedLease' + + ReclaimedLease: + type: object + description: One row a lease sweep reverted. + x-go-type: types.ReclaimedLease + x-go-type-import: + name: types + path: github.com/steveyegge/beads/internal/types + required: [id, previous_owner, revision] + properties: + id: + type: string + description: The reverted issue's id. + previous_owner: + type: string + description: >- + The assignee the stale lease was taken from; empty when the row + carried none. + revision: + type: string + description: >- + The row's NEW revision, minted by the revert: the + optimistic-concurrency token a guarded write after the reclaim + needs. A decimal string, like every `revision` on this surface. + SweepRequest: type: object additionalProperties: false diff --git a/internal/httpapi/spec_parity_test.go b/internal/httpapi/spec_parity_test.go index 3d9fb664d..3f7451ac1 100644 --- a/internal/httpapi/spec_parity_test.go +++ b/internal/httpapi/spec_parity_test.go @@ -420,11 +420,11 @@ func TestSpecDefaultsMatchSharedConstants(t *testing.T) { } } - // The id bound on the two request bodies that carry an `ids` list, for the - // anchor bound's reason: each is enforced in code — maxDeleteIDs in the - // delete handler, issueops.MaxGetManyIDs in the role behind batchGet — and - // restated as the member's maxItems, which is where a generated client - // learns it. Each body is reached through its operation's own $ref, so a + // The id bound on the three request bodies that carry an `ids` list, for + // the anchor bound's reason: each is enforced in code — maxDeleteIDs in the + // delete handler, issueops.MaxGetManyIDs in the role behind batchGet and + // issueops.MaxReclaimIDs in the role behind reclaim — and restated as the + // member's maxItems, which is where a generated client learns it. Each body is reached through its operation's own $ref, so a // row cannot go on checking a schema its operation stopped using. for _, tc := range []struct { opID string @@ -432,6 +432,7 @@ func TestSpecDefaultsMatchSharedConstants(t *testing.T) { }{ {OpBatchGetIssues, issueops.MaxGetManyIDs}, {OpDeleteIssues, maxDeleteIDs}, + {OpReclaimIssues, issueops.MaxReclaimIDs}, } { so, ok := ops[tc.opID] if !ok { diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 1bff0651a..a9c50dec3 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -3894,6 +3894,85 @@ "required": true, "item_type": "object" }, + { + "schema": "ReclaimIssuesRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "ReclaimIssuesRequest", + "member": "any_replica", + "type": "boolean", + "required": false + }, + { + "schema": "ReclaimIssuesRequest", + "member": "assignees", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ReclaimIssuesRequest", + "member": "exclude_labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ReclaimIssuesRequest", + "member": "ids", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ReclaimIssuesRequest", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ReclaimIssuesRequest", + "member": "labels_any", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ReclaimIssuesRequest", + "member": "older_than_seconds", + "type": "number", + "format": "double", + "required": false + }, + { + "schema": "ReclaimIssuesResult", + "member": "reclaimed", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "ReclaimedLease", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "ReclaimedLease", + "member": "previous_owner", + "type": "string", + "required": true + }, + { + "schema": "ReclaimedLease", + "member": "revision", + "type": "string", + "required": true + }, { "schema": "Ref", "member": "id", @@ -6125,6 +6204,14 @@ "QueryPage": "response", "ReadyCount": "response", "ReadyPage": "response", + "ReclaimIssuesRequest": "request", + "ReclaimIssuesRequest.assignees[]": "request", + "ReclaimIssuesRequest.exclude_labels[]": "request", + "ReclaimIssuesRequest.ids[]": "request", + "ReclaimIssuesRequest.labels[]": "request", + "ReclaimIssuesRequest.labels_any[]": "request", + "ReclaimIssuesResult": "response", + "ReclaimedLease": "response", "Ref": "request", "RelatedIssues": "response", "ReleaseIssueRequest": "request", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 8e8bc8be3..1f4e4a6ad 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -25,6 +25,7 @@ go_library( "dispatch.go", "errors.go", "journal.go", + "leasereclaimer.go", "lifecycle.go", "list_walk.go", "memories.go", @@ -135,6 +136,7 @@ go_test( "served_hooks_test.go", "served_issue_operations_test.go", "served_journal_test.go", + "served_lease_reclaimer_test.go", "served_lifecycle_create_test.go", "served_list_test.go", "served_memories_deps_test.go", diff --git a/internal/httpclient/accessors.go b/internal/httpclient/accessors.go index ee635ebd9..8a6298155 100644 --- a/internal/httpclient/accessors.go +++ b/internal/httpclient/accessors.go @@ -384,3 +384,12 @@ func (s *Store) BatchApplier() (issueops.BatchApplier, error) { func (s *Store) BatchGetter() (issueops.BatchGetter, error) { return &httpBatchGetter{store: s}, nil } + +// LeaseReclaimer serves reclaimIssues (POST /v0/beads/issues:reclaim) — the +// stale-lease sweep behind `bd reclaim`. Like BatchGetter it dials through the +// store directly rather than through roleWire: its request is one body with no +// path id, and Store.dispatch already preflights the capability and never +// retries a POST. See leasereclaimer.go. +func (s *Store) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return &httpLeaseReclaimer{store: s}, nil +} diff --git a/internal/httpclient/encode/oracle_test.go b/internal/httpclient/encode/oracle_test.go index d127a83c4..f65bb152a 100644 --- a/internal/httpclient/encode/oracle_test.go +++ b/internal/httpclient/encode/oracle_test.go @@ -71,6 +71,7 @@ func startOracle(t *testing.T) *oracle { EdgeReader: stubEdges{}, GraphCounter: stubGraphCounter{}, BatchGetter: stubBatchGetter{}, + LeaseReclaimer: stubLeaseReclaimer{}, Relations: stubRelations{}, Commenter: stubCommenter{}, BlockingAnnotator: stubBlocking{}, @@ -270,6 +271,7 @@ type ( stubEdges struct{ issueops.EdgeReader } stubGraphCounter struct{ issueops.GraphCounter } stubBatchGetter struct{ issueops.BatchGetter } + stubLeaseReclaimer struct{ issueops.LeaseReclaimer } stubRelations struct{ issueops.Relations } stubCommenter struct{ issueops.Commenter } stubBlocking struct{ issueops.BlockingAnnotator } diff --git a/internal/httpclient/leasereclaimer.go b/internal/httpclient/leasereclaimer.go new file mode 100644 index 000000000..5630fb351 --- /dev/null +++ b/internal/httpclient/leasereclaimer.go @@ -0,0 +1,135 @@ +package httpclient + +import ( + "context" + "net/http" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// httpLeaseReclaimer is issueops.LeaseReclaimer over reclaimIssues (POST +// /v0/beads/issues:reclaim). +// +// THE MAPPING IS TOTAL: every member of ReclaimRequest has a wire member, so +// nothing here refuses on shape and the ledger carries no row for this +// operation. The grace window travels as fractional seconds (a float64), which +// is exact for the whole-second and millisecond windows a reaper uses; a +// nanosecond-precise window may round. +// +// THE REQUEST RULES ARE CHECKED BEFORE THE DIAL, in the order +// internal/storage/issueops.ValidateReclaimRequest applies them, for +// checkGetManyIDs' reason: the server maps every role validation onto one +// generic invalid_value problem, so a round trip could never hand this client +// back the exact *issueops.TooManyReclaimIDsError{Requested, Cap} the role's +// contract promises. The rules are restated here rather than imported, because +// that package is the storage engine's shared body and does not belong in a +// transport client. +// +// AN EMPTY SCOPE SLICE IS NOT SENT. The server refuses a PRESENT, empty scope +// member (it would read as "no scope", the widening hazard bd reclaim refuses +// on the CLI), and on the role an empty slice and a nil one are the same +// request: no scope. Omitting it says exactly that. +type httpLeaseReclaimer struct{ store *Store } + +var _ issueops.LeaseReclaimer = (*httpLeaseReclaimer)(nil) + +// Reclaim dials POST /v0/beads/issues:reclaim. +func (l *httpLeaseReclaimer) Reclaim(ctx context.Context, req issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + if err := validateReclaimRequest(req); err != nil { + return issueops.ReclaimResult{}, err + } + + body := apigen.ReclaimIssuesRequest{Actor: req.Actor} + if req.OlderThan > 0 { + seconds := req.OlderThan.Seconds() + body.OlderThanSeconds = &seconds + } + scope := func(values []string) *[]string { + if len(values) == 0 { + return nil + } + // COPIED, never aliased: the role promises never to write through a + // caller's request. + copied := append([]string(nil), values...) + return &copied + } + body.Ids = scope(req.Filter.IDs) + body.Assignees = scope(req.Filter.Assignees) + body.Labels = scope(req.Filter.Labels) + body.LabelsAny = scope(req.Filter.LabelsAny) + body.ExcludeLabels = scope(req.Filter.ExcludeLabels) + if req.Filter.AnyReplica { + anyReplica := true + body.AnyReplica = &anyReplica + } + + var out apigen.ReclaimIssuesResult + if err := l.store.dispatch(ctx, wire.Request{ + Op: wire.OpReclaimIssues, + Method: http.MethodPost, + Path: wire.PathIssuesReclaim, + Body: body, + }, &out); err != nil { + return issueops.ReclaimResult{}, err + } + + reclaimed := make([]issueops.ReclaimedLease, 0, len(out.Reclaimed)) + for _, entry := range out.Reclaimed { + // The revision is carried as the wire's decimal string, the role's own + // spelling; it is read here only to refuse an answer whose token no + // guarded write could use. + if _, err := parseRevision("LeaseReclaimer.Reclaim", entry.Revision); err != nil { + return issueops.ReclaimResult{}, err + } + reclaimed = append(reclaimed, entry) + } + return issueops.ReclaimResult{Reclaimed: reclaimed}, nil +} + +// validateReclaimRequest restates the role's refusals, in the role's order and +// with the role's types: the actor (present, then within its column once +// trimmed), the grace window, the id cap (counted as sent), the first blank +// id, then the first blank entry of the other scopes. +func validateReclaimRequest(req issueops.ReclaimRequest) error { + field := func(name, format string, args ...any) error { + return &issueops.ReclaimFieldError{Field: name, Detail: invalid(format, args...).Error()} + } + actor := strings.TrimSpace(req.Actor) + if actor == "" { + return field(issueops.ReclaimFieldActor, "reclaim actor is required") + } + if err := types.CheckFieldLen("actor", actor); err != nil { + return field(issueops.ReclaimFieldActor, "%v", err) + } + if req.OlderThan < 0 { + return field(issueops.ReclaimFieldOlderThan, "reclaim older_than must not be negative") + } + if len(req.Filter.IDs) > issueops.MaxReclaimIDs { + return &issueops.TooManyReclaimIDsError{Requested: len(req.Filter.IDs), Cap: issueops.MaxReclaimIDs} + } + for i, id := range req.Filter.IDs { + if id == "" { + return field(issueops.ReclaimFieldIDs, "reclaim scope id at position %d is empty", i) + } + } + for _, scope := range []struct { + name string + values []string + }{ + {issueops.ReclaimFieldAssignees, req.Filter.Assignees}, + {issueops.ReclaimFieldLabels, req.Filter.Labels}, + {issueops.ReclaimFieldLabelsAny, req.Filter.LabelsAny}, + {issueops.ReclaimFieldExcludeLabels, req.Filter.ExcludeLabels}, + } { + for i, v := range scope.values { + if strings.TrimSpace(v) == "" { + return field(scope.name, "reclaim %s entry at position %d is blank", scope.name, i) + } + } + } + return nil +} diff --git a/internal/httpclient/served_harness_test.go b/internal/httpclient/served_harness_test.go index 917335a2a..8fe00ca67 100644 --- a/internal/httpclient/served_harness_test.go +++ b/internal/httpclient/served_harness_test.go @@ -297,6 +297,8 @@ func serveConfig(t *testing.T, s storage.DoltStorage) httpapi.Config { fail("Memories") cfg.BatchGetter, err = s.BatchGetter() fail("BatchGetter") + cfg.LeaseReclaimer, err = s.LeaseReclaimer() + fail("LeaseReclaimer") // THE JOURNAL IS NOT AN ACCESSOR, alone among the roles above: it is reached // by TYPE ASSERTION, because it is not on storage.DoltStorage and a backend diff --git a/internal/httpclient/served_lease_reclaimer_test.go b/internal/httpclient/served_lease_reclaimer_test.go new file mode 100644 index 000000000..8b23af9e9 --- /dev/null +++ b/internal/httpclient/served_lease_reclaimer_test.go @@ -0,0 +1,111 @@ +//go:build cgo + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" +) + +// The LeaseReclaimer contract, run through the http client against a real bd +// serve. Nothing is parked: every member of ReclaimRequest has a wire member, +// and every refusal the role lists is raised before the dial in the role's own +// order (leasereclaimer.go). +// +// The claim that writes each lease goes over the wire too, through the client's +// Claimer; only the aging of a lease and the attaching of a label reach past +// the wire, into the embedded database behind the server, because no verb can +// make a lease stale on demand. +// +// THE HOOK CASE runs the reclaimer off a HookFiringStore wrapped around the +// client, the chain cmd/bd composes for a connected workspace: bd serve runs no +// hooks, so it is the CLIENT's decorator that owes one on_update per reverted +// row, re-reading each row over the wire to hand the script. +func newServedLeaseReclaimerFixture(t *testing.T, prefix string) conformance.LeaseReclaimerFixture { + t.Helper() + env := newServedEnv(t, prefix) + reclaimer, err := env.subject.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + claimer, err := env.subject.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := reclaimer + if runner != nil { + if hooked, err = storage.NewHookFiringStore(env.subject, runner).LeaseReclaimer(); err != nil { + t.Fatalf("hooked LeaseReclaimer(): %v", err) + } + } + return conformance.LeaseReclaimerFixture{ + IssuePrefix: env.prefix, + LeaseReclaimer: reclaimer, + CreateIssue: env.createIssue, + Claimer: claimer, + Lifecycle: lifecycle, + Exec: env.exec, + QueryScalar: env.queryScalar, + HookedLeaseReclaimer: hooked, + FiredUpdates: fired, + } +} + +func TestServedLeaseReclaimerRevertsAStaleLease(t *testing.T) { + conformance.RunLeaseReclaimerRevertsAStaleLease(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr00")) +} + +func TestServedLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t *testing.T) { + conformance.RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr01")) +} + +func TestServedLeaseReclaimerLeavesALiveLeaseAlone(t *testing.T) { + conformance.RunLeaseReclaimerLeavesALiveLeaseAlone(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr02")) +} + +func TestServedLeaseReclaimerHonorsTheGraceWindow(t *testing.T) { + conformance.RunLeaseReclaimerHonorsTheGraceWindow(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr03")) +} + +func TestServedLeaseReclaimerScopesToTheNamedIDs(t *testing.T) { + conformance.RunLeaseReclaimerScopesToTheNamedIDs(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr04")) +} + +func TestServedLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t *testing.T) { + conformance.RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr05")) +} + +func TestServedLeaseReclaimerScopesToAssigneesAndLabels(t *testing.T) { + conformance.RunLeaseReclaimerScopesToAssigneesAndLabels(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr06")) +} + +func TestServedLeaseReclaimerAttributesTheRecoveryEvent(t *testing.T) { + conformance.RunLeaseReclaimerAttributesTheRecoveryEvent(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr07")) +} + +func TestServedLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t *testing.T) { + conformance.RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr08")) +} + +func TestServedLeaseReclaimerRefusesAMalformedRequest(t *testing.T) { + conformance.RunLeaseReclaimerRefusesAMalformedRequest(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr09")) +} + +func TestServedLeaseReclaimerRefusesABlankScopeEntry(t *testing.T) { + conformance.RunLeaseReclaimerRefusesABlankScopeEntry(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr12")) +} + +func TestServedLeaseReclaimerAcceptsExactlyTheCap(t *testing.T) { + conformance.RunLeaseReclaimerAcceptsExactlyTheCap(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr10")) +} + +func TestServedLeaseReclaimerDoesNotMutateTheCallerRequest(t *testing.T) { + conformance.RunLeaseReclaimerDoesNotMutateTheCallerRequest(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr11")) +} diff --git a/internal/httpclient/served_test.go b/internal/httpclient/served_test.go index 5037cb1f7..23b16b02e 100644 --- a/internal/httpclient/served_test.go +++ b/internal/httpclient/served_test.go @@ -240,6 +240,9 @@ func serveRoles(store storage.DoltStorage) (httpapi.Config, error) { if cfg.BatchGetter, err = store.BatchGetter(); err != nil { return cfg, err } + if cfg.LeaseReclaimer, err = store.LeaseReclaimer(); err != nil { + return cfg, err + } return cfg, nil } diff --git a/internal/httpclient/unsupported_gen.go b/internal/httpclient/unsupported_gen.go index 79e696c54..2e8cde872 100644 --- a/internal/httpclient/unsupported_gen.go +++ b/internal/httpclient/unsupported_gen.go @@ -742,4 +742,4 @@ func (unsupportedDoltStorage) VersionReconciler() (_ issueops.VersionReconciler, return } -// NOTE: partial shell (145 of 199 methods generated; 54 left to this package's hand-written set). +// NOTE: partial shell (145 of 200 methods generated; 55 left to this package's hand-written set). diff --git a/internal/httpclient/wave2b_roles_test.go b/internal/httpclient/wave2b_roles_test.go index 86c053aca..c64817ac1 100644 --- a/internal/httpclient/wave2b_roles_test.go +++ b/internal/httpclient/wave2b_roles_test.go @@ -8,8 +8,10 @@ import ( "reflect" "strings" "testing" + "time" "github.com/steveyegge/beads/internal/httpapi/apigen" + storeops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/workapi" "github.com/steveyegge/beads/issueops" @@ -542,3 +544,87 @@ func TestRelatedLeavesTheCallersTypesAlone(t *testing.T) { t.Errorf("the caller's Types changed across the call: %v, want %v", types_, snapshot) } } + +// TestReclaimValidationMatchesTheSharedValidator binds leasereclaimer.go's +// restated rules to internal/storage/issueops.ValidateReclaimRequest, for +// TestReleaseValidationMatchesTheSharedValidator's reason: a restated rule is +// how a leg drifts. It compares VERDICTS, and the cap's type, over a table that +// covers both sides of every rule. +func TestReclaimValidationMatchesTheSharedValidator(t *testing.T) { + overCap := make([]string, issueops.MaxReclaimIDs+1) + for name, req := range map[string]issueops.ReclaimRequest{ + "valid bare": {Actor: "reaper"}, + "valid scoped": {Actor: "reaper", OlderThan: time.Minute, Filter: issueops.ReclaimFilter{IDs: []string{"bd-1"}}}, + "no actor": {}, + "blank actor": {Actor: " "}, + "negative grace": {Actor: "reaper", OlderThan: -time.Second}, + "blank id": {Actor: "reaper", Filter: issueops.ReclaimFilter{IDs: []string{"bd-1", ""}}}, + "padded id is an id": {Actor: "reaper", Filter: issueops.ReclaimFilter{IDs: []string{" bd-1 "}}}, + "over the cap": {Actor: "reaper", Filter: issueops.ReclaimFilter{IDs: overCap}}, + "blank assignee": {Actor: "reaper", Filter: issueops.ReclaimFilter{Assignees: []string{" "}}}, + "blank label": {Actor: "reaper", Filter: issueops.ReclaimFilter{Labels: []string{"a", ""}}}, + "blank label-any": {Actor: "reaper", Filter: issueops.ReclaimFilter{LabelsAny: []string{"\t"}}}, + "blank exclude": {Actor: "reaper", Filter: issueops.ReclaimFilter{ExcludeLabels: []string{""}}}, + + // The actor is held to its column once trimmed, as the sweep records it. + "actor at the bound": {Actor: strings.Repeat("r", types.MaxFieldLen)}, + "padded actor at the bound": {Actor: " " + strings.Repeat("r", types.MaxFieldLen) + " "}, + "over-long actor": {Actor: strings.Repeat("r", types.MaxFieldLen+1)}, + } { + t.Run(name, func(t *testing.T) { + mine := validateReclaimRequest(req) + shared := storeops.ValidateReclaimRequest(req) + if (mine != nil) != (shared != nil) { + t.Fatalf("validateReclaimRequest = %v, storeops.ValidateReclaimRequest = %v: the client's restatement has drifted", mine, shared) + } + var mineCap, sharedCap *issueops.TooManyReclaimIDsError + if errors.As(mine, &mineCap) != errors.As(shared, &sharedCap) { + t.Fatalf("cap type differs: client %v, shared %v", mine, shared) + } + var mineField, sharedField *issueops.ReclaimFieldError + errors.As(mine, &mineField) + errors.As(shared, &sharedField) + if (mineField == nil) != (sharedField == nil) || (mineField != nil && mineField.Field != sharedField.Field) { + t.Fatalf("field differs: client %v, shared %v", mine, shared) + } + }) + } +} + +// TestReclaimSendsAnyReplicaOnlyWhenAsked pins the one filter member that +// WIDENS a sweep rather than narrowing it. ReclaimFilter.AnyReplica disarms the +// granting-replica guard, so a client that sent it unasked would revert leases +// another replica granted, and one that dropped it would keep the guard a caller +// had disarmed. Unasked it must be absent from the body; asked it must be true. +func TestReclaimSendsAnyReplicaOnlyWhenAsked(t *testing.T) { + for name, anyReplica := range map[string]bool{"unasked": false, "asked": true} { + t.Run(name, func(t *testing.T) { + store, w := recordingStore(t) + role, err := store.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + if _, err := role.Reclaim(context.Background(), issueops.ReclaimRequest{ + Actor: "reaper", + Filter: issueops.ReclaimFilter{IDs: []string{"bd-1"}, AnyReplica: anyReplica}, + }); err != nil { + t.Fatalf("Reclaim(): %v", err) + } + if len(w.requests) != 1 { + t.Fatalf("dialed %d times, want 1", len(w.requests)) + } + body, ok := w.requests[0].Body.(apigen.ReclaimIssuesRequest) + if !ok { + t.Fatalf("the body is %T, want the generated ReclaimIssuesRequest", w.requests[0].Body) + } + switch { + case !anyReplica && body.AnyReplica != nil: + t.Errorf("an unasked sweep sent any_replica=%v; the guard must stay armed", *body.AnyReplica) + case anyReplica && body.AnyReplica == nil: + t.Error("a sweep that disarmed the replica guard sent no any_replica; the server would keep it armed") + case anyReplica && !*body.AnyReplica: + t.Error("a sweep that disarmed the replica guard sent any_replica=false, want true") + } + }) + } +} diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index b7db819c2..60476683a 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -102,6 +102,11 @@ var opCapability = map[string]string{ // the block above states: the server publishes it ahead of the accessor // that dials it, and the set-equality gate needs its token here first. OpBatchGetIssues: "issues.batchGet", + // The stale-lease sweep, dialed by httpLeaseReclaimer. A server that + // predates it does not advertise the token, so Preflight refuses locally + // with a typed capability error instead of dialing a path the older server + // would answer as a claim of an issue called ":reclaim". + OpReclaimIssues: "issues.reclaim", } // CapProjectEnforce is the behavior capability the server advertises to announce diff --git a/internal/httpclient/wire/paths.go b/internal/httpclient/wire/paths.go index 6357c526d..94a90b83a 100644 --- a/internal/httpclient/wire/paths.go +++ b/internal/httpclient/wire/paths.go @@ -57,6 +57,7 @@ const ( OpGetMemory = "getMemory" OpForgetMemory = "forgetMemory" OpBatchGetIssues = "batchGetIssues" + OpReclaimIssues = "reclaimIssues" ) // The paths that carry no caller-supplied segment. A path with one is built by @@ -75,6 +76,7 @@ const ( PathIssuesBatchCreate = "/v0/beads/issues:batchCreate" PathIssuesBatchClose = "/v0/beads/issues:batchClose" PathIssuesBatchGet = "/v0/beads/issues:batchGet" + PathIssuesReclaim = "/v0/beads/issues:reclaim" PathIssuesClaimNext = "/v0/beads/issues:claimNext" PathIssuesBatchApply = "/v0/beads/issues:batchApply" PathSettings = "/v0/beads/config" diff --git a/internal/httpclient/wire/revision_tolerance_test.go b/internal/httpclient/wire/revision_tolerance_test.go index 9b410298f..1c9f33f16 100644 --- a/internal/httpclient/wire/revision_tolerance_test.go +++ b/internal/httpclient/wire/revision_tolerance_test.go @@ -283,6 +283,10 @@ var revisionSchemaExempt = map[string]string{ "#6053 and types.BatchGetIssue declares Revision a string, so every server that routes it " + "answers the decimal-string shape; a pre-#6053 server never advertises issues.batchGet, so " + "Preflight refuses the operation with a *CapabilityError before any body is decoded", + "ReclaimedLease": "ReclaimIssuesResult.reclaimed's element. issues:reclaim postdates #6053 and " + + "types.ReclaimedLease declares Revision a string, so every server that routes it answers the " + + "decimal-string shape; a pre-#6053 server never advertises issues.reclaim, so Preflight refuses " + + "the operation with a *CapabilityError before any body is decoded", } // TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember is diff --git a/internal/httpclient/wire/skew_test.go b/internal/httpclient/wire/skew_test.go index 04b023171..fe2fc7ace 100644 --- a/internal/httpclient/wire/skew_test.go +++ b/internal/httpclient/wire/skew_test.go @@ -110,16 +110,17 @@ func TestAgainstAFirstSliceServerEveryPostBaselineOperationRefusesWithCaseTwoDat t.Fatalf("post-baseline ops on a first-slice token = %v, want exactly [claimIssue]", compat) } // The dimension is pinned so the matrix cannot shrink silently: the first - // slice served six operations and the tip surface has forty-two, so - // thirty-six of them are things a first-slice server has never heard of. The + // slice served six operations and the tip surface has forty-three, so + // thirty-seven of them are things a first-slice server has never heard of. The // wire wave moved this from twenty-seven to thirty — claimNext, release and // count — wave 2 to thirty-two, adding the dependency-edge count and the // related read, the mini-sync to thirty-five: the comment write and the two - // config writes, and the batch read (upstream #7248) to thirty-six. Each is + // config writes, the batch read (upstream #7248) to thirty-six, and the + // stale-lease sweep to thirty-seven. Each is // an operation a first-slice server never served and this client does not // yet dial, which is exactly the shape the refusal matrix is about. - if len(refused) != 36 { - t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 36", len(refused)) + if len(refused) != 37 { + t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 37", len(refused)) } // The compat op forces the handshake yet preflights clean: issues.claim is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 23d98908d..c650ac7f5 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -68,6 +68,7 @@ func TestOperationIdsMatchTheServerRouteTable(t *testing.T) { {OpListEvents, httpapi.OpListEvents}, {OpWatchEvents, httpapi.OpWatchEvents}, {OpBatchGetIssues, httpapi.OpBatchGetIssues}, + {OpReclaimIssues, httpapi.OpReclaimIssues}, } // The table is a HAND LIST, and a hand list's failure mode is omission: a new diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index 86a1ef4f6..e4a8b5a27 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -38,6 +38,7 @@ go_library( "hook_issue_claimer.go", "hook_issue_operations.go", "hook_issue_reader.go", + "hook_lease_reclaimer.go", "hook_memories.go", "hook_metadata_cas.go", "hook_querier.go", @@ -90,6 +91,7 @@ go_test( "hook_issue_claimer_test.go", "hook_issue_operations_test.go", "hook_issue_reader_test.go", + "hook_lease_reclaimer_test.go", "hook_metadata_cas_test.go", "hook_ready_claimer_test.go", "hook_relations_test.go", diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 989f2d8e4..1f79aa088 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -47,6 +47,7 @@ go_library( "iter_stubs.go", "journal_accessor.go", "labels.go", + "lease_reclaimer.go", "memories.go", "merge_slot.go", "metadata_cas.go", @@ -268,6 +269,7 @@ go_test( "large_batch_apply_measure_test.go", "large_graph_create_bench_test.go", "lease_migration_test.go", + "lease_reclaimer_contract_test.go", "lease_test.go", "lenient_open_dirty_tables_integration_test.go", "lifecycle_close_reopen_contract_test.go", diff --git a/internal/storage/dolt/lease_reclaimer.go b/internal/storage/dolt/lease_reclaimer.go new file mode 100644 index 000000000..492a47f48 --- /dev/null +++ b/internal/storage/dolt/lease_reclaimer.go @@ -0,0 +1,50 @@ +package dolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// LeaseReclaimer returns the lease-sweep surface for this store. +func (s *DoltStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + if s == nil { + return nil, &storage.ErrUnsupported{Op: "LeaseReclaimer", Backend: "nil"} + } + return &leaseReclaimer{store: s}, nil +} + +// leaseReclaimer answers a Reclaim from one retrying write transaction around +// the shared body, storeops.ExecuteReclaimInTx. +// +// THE VERSION-CONTROL ENTRY GOES THROUGH runIssueOperationTxWithMessage, the +// funnel Releaser uses, rather than the raw ReclaimExpiredLeases' in-transaction +// DOLT_COMMIT: the funnel is what honors a deferred version commit on the +// context (dolt.auto-commit batch/off), which a role a front door calls has to, +// and a sweep that reverted nothing composes no staged set and so records no +// entry describing a no-op. +type leaseReclaimer struct{ store *DoltStore } + +var _ issueops.LeaseReclaimer = (*leaseReclaimer)(nil) + +func (l *leaseReclaimer) Reclaim(ctx context.Context, request issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + if err := storeops.ValidateReclaimRequest(request); err != nil { + return issueops.ReclaimResult{}, err + } + var result issueops.ReclaimResult + if err := l.store.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + var err error + result, err = storeops.ExecuteReclaimInTx(ctx, tx, request) + if err != nil { + return nil, "", err + } + tables, msg := storeops.ReclaimVersionCommit(result) + return tables, msg, nil + }); err != nil { + return issueops.ReclaimResult{}, err + } + return result, nil +} diff --git a/internal/storage/dolt/lease_reclaimer_contract_test.go b/internal/storage/dolt/lease_reclaimer_contract_test.go new file mode 100644 index 000000000..cfc19f3e9 --- /dev/null +++ b/internal/storage/dolt/lease_reclaimer_contract_test.go @@ -0,0 +1,114 @@ +package dolt + +import ( + "context" + "fmt" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" +) + +// TestLeaseReclaimerContract runs the LeaseReclaimer contract against the +// server-backed store, which wraps internal/storage/issueops.ExecuteReclaimInTx +// in its own retrying write transaction. Like every role suite here the cases +// share one store, so each namespaces its seeds and scopes every sweep. +func TestLeaseReclaimerContract(t *testing.T) { + fixture, ctx, cleanup := newDoltLeaseReclaimerFixture(t, "lrc") + defer cleanup() + + t.Run("RevertsAStaleLease", func(t *testing.T) { + conformance.RunLeaseReclaimerRevertsAStaleLease(t, ctx, fixture) + }) + t.Run("MintsARevisionThatFencesTheFormerHolder", func(t *testing.T) { + conformance.RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t, ctx, fixture) + }) + t.Run("LeavesALiveLeaseAlone", func(t *testing.T) { + conformance.RunLeaseReclaimerLeavesALiveLeaseAlone(t, ctx, fixture) + }) + t.Run("HonorsTheGraceWindow", func(t *testing.T) { + conformance.RunLeaseReclaimerHonorsTheGraceWindow(t, ctx, fixture) + }) + t.Run("ScopesToTheNamedIDs", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToTheNamedIDs(t, ctx, fixture) + }) + t.Run("AnswersAnAbsentIDWithAnEmptyResult", func(t *testing.T) { + conformance.RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t, ctx, fixture) + }) + t.Run("ScopesToAssigneesAndLabels", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToAssigneesAndLabels(t, ctx, fixture) + }) + t.Run("AttributesTheRecoveryEvent", func(t *testing.T) { + conformance.RunLeaseReclaimerAttributesTheRecoveryEvent(t, ctx, fixture) + }) + t.Run("FiresOnUpdateOncePerReclaimedRow", func(t *testing.T) { + conformance.RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t, ctx, fixture) + }) + t.Run("RefusesAMalformedRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesAMalformedRequest(t, ctx, fixture) + }) + t.Run("RefusesABlankScopeEntry", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesABlankScopeEntry(t, ctx, fixture) + }) + t.Run("AcceptsExactlyTheCap", func(t *testing.T) { + conformance.RunLeaseReclaimerAcceptsExactlyTheCap(t, ctx, fixture) + }) + t.Run("DoesNotMutateTheCallerRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerDoesNotMutateTheCallerRequest(t, ctx, fixture) + }) +} + +// newDoltLeaseReclaimerFixture composes the frozen role kit with this +// backend's accessors, plus the same reclaimer taken off a HookFiringStore so +// the hook case observes the decorator a command actually holds. +func newDoltLeaseReclaimerFixture(t *testing.T, prefix string) (conformance.LeaseReclaimerFixture, context.Context, func()) { + t.Helper() + store, storeCleanup := setupTestStore(t) + ctx, cancel := testContext(t) + stop := func() { + cancel() + storeCleanup() + } + fail := func(what string, err error) { + stop() + t.Fatalf("%s: %v", what, err) + } + reclaimer, err := store.LeaseReclaimer() + if err != nil { + fail("LeaseReclaimer()", err) + } + claimer, err := store.IssueClaimer() + if err != nil { + fail("IssueClaimer()", err) + } + lifecycle, err := store.IssueLifecycle() + if err != nil { + fail("IssueLifecycle()", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := reclaimer + if runner != nil { + if hooked, err = storage.NewHookFiringStore(store, runner).LeaseReclaimer(); err != nil { + fail("hooked LeaseReclaimer()", err) + } + } + kit := newDoltRoleFixtureKit(store, prefix) + return conformance.LeaseReclaimerFixture{ + IssuePrefix: kit.IssuePrefix, + LeaseReclaimer: reclaimer, + CreateIssue: kit.CreateIssue, + Claimer: claimer, + Lifecycle: lifecycle, + QueryScalar: kit.QueryScalar, + Exec: func(ctx context.Context, statements []conformance.SQLStatement) error { + for _, stmt := range statements { + if _, err := store.db.ExecContext(ctx, stmt.Query, stmt.Args...); err != nil { + return fmt.Errorf("%s: %w", stmt.Query, err) + } + } + return nil + }, + HookedLeaseReclaimer: hooked, + FiredUpdates: fired, + }, ctx, stop +} diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index a9d2e3c7a..4a1282e25 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -1309,4 +1309,11 @@ func (r *issueSQLRepositoryImpl) ReclaimExpiredLeases(ctx context.Context, older return out, nil } +// Reclaim runs the shared sweep body directly on r.runner, matching GetMany's +// bare-passthrough shape: validation and error-wrapping both happen inside +// issueops.ExecuteReclaimInTx. +func (r *issueSQLRepositoryImpl) Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { + return issueops.ExecuteReclaimInTx(ctx, r.runner, request) +} + const deleteBatchSize = 200 diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index a6ce46a83..7f5ec7208 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -117,6 +117,11 @@ type IssueSQLRepository interface { UnclaimIssueIfAssignee(ctx context.Context, id, actor, expectedAssignee string) error HeartbeatIssue(ctx context.Context, id, actor string) error ReclaimExpiredLeases(ctx context.Context, olderThan time.Duration, filter types.ReclaimFilter, actor string) ([]types.ReclaimedLease, error) + // Reclaim runs the SHARED sweep body (issueops.ExecuteReclaimInTx) on this + // repository's transaction, which is how the unit-of-work provider reaches + // the same function the two store backends wrap. It is the publicops-typed + // counterpart to ReclaimExpiredLeases above, the way GetMany is to GetByIDs. + Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) WakeExpiredDefers(ctx context.Context) (issues, wisps int, err error) } @@ -344,6 +349,10 @@ type IssueUseCase interface { UnclaimIfAssignee(ctx context.Context, id, actor, expectedAssignee string) error Heartbeat(ctx context.Context, id, actor string) error ReclaimExpiredLeases(ctx context.Context, olderThan time.Duration, filter types.ReclaimFilter, actor string) ([]types.ReclaimedLease, error) + // Reclaim is the shape issueops.LeaseReclaimer publishes; see + // IssueRepository.Reclaim for why it is a separate method from + // ReclaimExpiredLeases rather than a replacement for it. + Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) WakeExpiredDefers(ctx context.Context) (issues, wisps int, err error) CreateIssue(ctx context.Context, params CreateIssueParams, actor string) (CreateIssueResult, error) @@ -1978,3 +1987,11 @@ func (u *issueUseCaseImpl) ReclaimExpiredLeases(ctx context.Context, olderThan t } return out, nil } + +// Reclaim passes straight through to the repository with no pre-check and no +// error-wrapping, matching GetMany's reasoning: the request is fully +// validated inside the shared body (issueops.ExecuteReclaimInTx), and there is +// no use-case-level concern layered on top of it. +func (u *issueUseCaseImpl) Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { + return u.issueRepo.Reclaim(ctx, request) +} diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 64e516d60..8241d7414 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -37,6 +37,7 @@ go_library( "iter_stubs.go", "journal_accessor.go", "labels.go", + "lease_reclaimer.go", "list_queries.go", "memories.go", "merge_slot.go", @@ -169,6 +170,7 @@ go_test( "label_rename_test.go", "large_batch_apply_measure_test.go", "large_graph_create_bench_test.go", + "lease_reclaimer_contract_test.go", "lease_replica_test.go", "lease_test.go", "lenient_gate_warning_test.go", diff --git a/internal/storage/embeddeddolt/lease_reclaimer.go b/internal/storage/embeddeddolt/lease_reclaimer.go new file mode 100644 index 000000000..cfbfb1c28 --- /dev/null +++ b/internal/storage/embeddeddolt/lease_reclaimer.go @@ -0,0 +1,54 @@ +//go:build cgo + +package embeddeddolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// LeaseReclaimer returns the lease-sweep surface for this store. +func (s *EmbeddedDoltStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + if s == nil { + return nil, &storage.ErrUnsupported{Op: "LeaseReclaimer", Backend: "nil"} + } + return &leaseReclaimer{store: s}, nil +} + +// leaseReclaimer answers a Reclaim from one connection's transaction around the +// shared body, storeops.ExecuteReclaimInTx, the embedded twin of the +// server-backed store's leaseReclaimer. +// +// THE VERSION-CONTROL ENTRY LANDS AFTER THE SQL COMMIT, through +// runIssueOperationTxWithMessage, Releaser's funnel: unlike the raw +// ReclaimExpiredLeases, which leaves its rows in the working set for the CLI's +// commitPendingIfEmbedded, a role is called by front doors (bd serve, a +// library caller) that have no such second step, so the role records its own +// entry, and the funnel is what lets a deferred version commit on the context +// suppress it. +type leaseReclaimer struct{ store *EmbeddedDoltStore } + +var _ issueops.LeaseReclaimer = (*leaseReclaimer)(nil) + +func (l *leaseReclaimer) Reclaim(ctx context.Context, request issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + if err := storeops.ValidateReclaimRequest(request); err != nil { + return issueops.ReclaimResult{}, err + } + var result issueops.ReclaimResult + if err := l.store.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + var err error + result, err = storeops.ExecuteReclaimInTx(ctx, tx, request) + if err != nil { + return nil, "", err + } + tables, msg := storeops.ReclaimVersionCommit(result) + return tables, msg, nil + }); err != nil { + return issueops.ReclaimResult{}, err + } + return result, nil +} diff --git a/internal/storage/embeddeddolt/lease_reclaimer_contract_test.go b/internal/storage/embeddeddolt/lease_reclaimer_contract_test.go new file mode 100644 index 000000000..d834b3f92 --- /dev/null +++ b/internal/storage/embeddeddolt/lease_reclaimer_contract_test.go @@ -0,0 +1,111 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "fmt" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" +) + +// TestLeaseReclaimerContract runs the LeaseReclaimer contract against the +// embedded store, which wraps internal/storage/issueops.ExecuteReclaimInTx in +// one connection's transaction and records its version entry after the SQL +// commit. +func TestLeaseReclaimerContract(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "lrc") + ctx := t.Context() + fixture := newEmbeddedLeaseReclaimerFixture(t, te, "lrc") + + t.Run("RevertsAStaleLease", func(t *testing.T) { + conformance.RunLeaseReclaimerRevertsAStaleLease(t, ctx, fixture) + }) + t.Run("MintsARevisionThatFencesTheFormerHolder", func(t *testing.T) { + conformance.RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t, ctx, fixture) + }) + t.Run("LeavesALiveLeaseAlone", func(t *testing.T) { + conformance.RunLeaseReclaimerLeavesALiveLeaseAlone(t, ctx, fixture) + }) + t.Run("HonorsTheGraceWindow", func(t *testing.T) { + conformance.RunLeaseReclaimerHonorsTheGraceWindow(t, ctx, fixture) + }) + t.Run("ScopesToTheNamedIDs", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToTheNamedIDs(t, ctx, fixture) + }) + t.Run("AnswersAnAbsentIDWithAnEmptyResult", func(t *testing.T) { + conformance.RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t, ctx, fixture) + }) + t.Run("ScopesToAssigneesAndLabels", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToAssigneesAndLabels(t, ctx, fixture) + }) + t.Run("AttributesTheRecoveryEvent", func(t *testing.T) { + conformance.RunLeaseReclaimerAttributesTheRecoveryEvent(t, ctx, fixture) + }) + t.Run("FiresOnUpdateOncePerReclaimedRow", func(t *testing.T) { + conformance.RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t, ctx, fixture) + }) + t.Run("RefusesAMalformedRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesAMalformedRequest(t, ctx, fixture) + }) + t.Run("RefusesABlankScopeEntry", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesABlankScopeEntry(t, ctx, fixture) + }) + t.Run("AcceptsExactlyTheCap", func(t *testing.T) { + conformance.RunLeaseReclaimerAcceptsExactlyTheCap(t, ctx, fixture) + }) + t.Run("DoesNotMutateTheCallerRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerDoesNotMutateTheCallerRequest(t, ctx, fixture) + }) +} + +func newEmbeddedLeaseReclaimerFixture(t *testing.T, te *testEnv, prefix string) conformance.LeaseReclaimerFixture { + t.Helper() + reclaimer, err := te.store.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + claimer, err := te.store.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + lifecycle, err := te.store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := reclaimer + if runner != nil { + if hooked, err = storage.NewHookFiringStore(te.store, runner).LeaseReclaimer(); err != nil { + t.Fatalf("hooked LeaseReclaimer(): %v", err) + } + } + kit := newEmbeddedRoleFixtureKit(te, prefix) + return conformance.LeaseReclaimerFixture{ + IssuePrefix: kit.IssuePrefix, + LeaseReclaimer: reclaimer, + CreateIssue: kit.CreateIssue, + Claimer: claimer, + Lifecycle: lifecycle, + QueryScalar: kit.QueryScalar, + Exec: func(ctx context.Context, statements []conformance.SQLStatement) error { + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + return err + } + defer func() { _ = cleanup() }() + for _, stmt := range statements { + if _, err := db.ExecContext(ctx, stmt.Query, stmt.Args...); err != nil { + return fmt.Errorf("%s: %w", stmt.Query, err) + } + } + return nil + }, + HookedLeaseReclaimer: hooked, + FiredUpdates: fired, + } +} diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 0b9e9293b..f7826b43e 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -119,7 +119,10 @@ func (p *uowProvider) VersionReconciler() (publicops.VersionReconciler, error) { } func (p *uowProvider) MetadataCAS() (publicops.MetadataCAS, error) { return uow.NewMetadataCAS(p) } func (p *uowProvider) Releaser() (publicops.Releaser, error) { return uow.NewReleaser(p) } -func (p *uowProvider) Memories() (memoryops.Memories, error) { return uow.NewMemories(p) } +func (p *uowProvider) LeaseReclaimer() (publicops.LeaseReclaimer, error) { + return uow.NewLeaseReclaimer(p) +} +func (p *uowProvider) Memories() (memoryops.Memories, error) { return uow.NewMemories(p) } func (p *uowProvider) EventsJournalCursor() (storage.EventsJournalCursor, error) { return uow.NewEventsJournalCursor(p) } @@ -195,6 +198,7 @@ var ( _ uow.VersionReconcilerSource = (*uowProvider)(nil) _ uow.MetadataCASSource = (*uowProvider)(nil) _ uow.ReleaserSource = (*uowProvider)(nil) + _ uow.LeaseReclaimerSource = (*uowProvider)(nil) _ uow.MemoriesSource = (*uowProvider)(nil) _ uow.EventsJournalCursorSource = (*uowProvider)(nil) ) diff --git a/internal/storage/hook_batch_applier_test.go b/internal/storage/hook_batch_applier_test.go index 92844911c..a211e0908 100644 --- a/internal/storage/hook_batch_applier_test.go +++ b/internal/storage/hook_batch_applier_test.go @@ -39,6 +39,9 @@ func (r *recordingBatchApplyHooks) CompleteIssueOperationMetadata(_ context.Cont func (r *recordingBatchApplyHooks) CompleteIssueOperationRelease(_ context.Context, issueID string) { r.completions = append(r.completions, "release:"+issueID) } +func (r *recordingBatchApplyHooks) CompleteIssueOperationReclaim(_ context.Context, issueID string) { + r.completions = append(r.completions, "reclaim:"+issueID) +} func issueIDOrNil(issue *types.Issue) string { if issue == nil { @@ -230,3 +233,4 @@ func (r *recordingBatchApplyIssues) CompleteIssueOperationDependency(context.Con func (r *recordingBatchApplyIssues) CompleteIssueOperationComment(context.Context, string) {} func (r *recordingBatchApplyIssues) CompleteIssueOperationMetadata(context.Context, string) {} func (r *recordingBatchApplyIssues) CompleteIssueOperationRelease(context.Context, string) {} +func (r *recordingBatchApplyIssues) CompleteIssueOperationReclaim(context.Context, string) {} diff --git a/internal/storage/hook_decorator.go b/internal/storage/hook_decorator.go index c0342252e..c6be30885 100644 --- a/internal/storage/hook_decorator.go +++ b/internal/storage/hook_decorator.go @@ -153,6 +153,13 @@ func RoleFiresHooks(role any) bool { // hazard, on the operation that reaches it with nothing else in the way. case *hookBatchCreator: return true + // The lease reclaimer fires the update hook once per row it reverts + // (hook_lease_reclaimer.go), so a scheduled reaper sweep is N + // subprocesses per run — the same value Releaser and ReadyClaimer are + // refused for, and the one whose caller is a clock rather than a human + // or a request. + case *hookLeaseReclaimer: + return true } return false } @@ -466,6 +473,15 @@ func (h *HookFiringStore) CompleteIssueOperationRelease(ctx context.Context, iss h.fireHookByID(ctx, hooks.EventUpdate, issueID) } +// CompleteIssueOperationReclaim fires the update hook for one row a lease +// sweep reverted, which is the event the generic update path fires for the +// same write — assignee and status are exactly the fields a reclaim moves, +// the same pair a release moves. It re-reads the row rather than taking the +// caller's, so a script sees the unheld issue the sweep produced. +func (h *HookFiringStore) CompleteIssueOperationReclaim(ctx context.Context, issueID string) { + h.fireHookByID(ctx, hooks.EventUpdate, issueID) +} + func (h *HookFiringStore) fireHookByID(ctx context.Context, event, id string) { if h.runner == nil { return diff --git a/internal/storage/hook_issue_operations.go b/internal/storage/hook_issue_operations.go index ba41e5d81..7926a5694 100644 --- a/internal/storage/hook_issue_operations.go +++ b/internal/storage/hook_issue_operations.go @@ -35,6 +35,7 @@ type issueOperationHooks interface { CompleteIssueOperationComment(ctx context.Context, issueID string) CompleteIssueOperationMetadata(ctx context.Context, issueID string) CompleteIssueOperationRelease(ctx context.Context, issueID string) + CompleteIssueOperationReclaim(ctx context.Context, issueID string) } var _ issueOperationHooks = (*HookFiringStore)(nil) diff --git a/internal/storage/hook_issue_operations_test.go b/internal/storage/hook_issue_operations_test.go index 0c86b95ea..13a788060 100644 --- a/internal/storage/hook_issue_operations_test.go +++ b/internal/storage/hook_issue_operations_test.go @@ -63,6 +63,9 @@ func (r *recordingIssueOperationHooks) CompleteIssueOperationMetadata(_ context. func (r *recordingIssueOperationHooks) CompleteIssueOperationRelease(_ context.Context, issueID string) { r.completions = append(r.completions, "release:"+issueID) } +func (r *recordingIssueOperationHooks) CompleteIssueOperationReclaim(_ context.Context, issueID string) { + r.completions = append(r.completions, "reclaim:"+issueID) +} // lifecycleStore is a DoltStorage whose only real method is IssueLifecycle. type lifecycleStore struct { diff --git a/internal/storage/hook_lease_reclaimer.go b/internal/storage/hook_lease_reclaimer.go new file mode 100644 index 000000000..e94877f93 --- /dev/null +++ b/internal/storage/hook_lease_reclaimer.go @@ -0,0 +1,42 @@ +package storage + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// LeaseReclaimer returns the inner store's lease-sweep surface with this +// decorator's completion hooks layered over it. +// +// It WRAPS rather than recursing unwrapped, for the reason Releaser does: a +// reclaim changes an issue's assignee and status per reverted row, which is +// an on_update in the vocabulary internal/hooks publishes, and the row it +// names is still there to hand a script. +// +// It recurses instead of delegating, so a blind delegation would hand back +// the inner store's surface and silently stop firing it. +func (h *HookFiringStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + inner, err := h.inner.LeaseReclaimer() + if err != nil { + return nil, err + } + return &hookLeaseReclaimer{inner: inner, hooks: h}, nil +} + +type hookLeaseReclaimer struct { + inner issueops.LeaseReclaimer + hooks issueOperationHooks +} + +// Reclaim fires the update hook once for every row the sweep reverted. +func (r *hookLeaseReclaimer) Reclaim(ctx context.Context, request issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + result, err := r.inner.Reclaim(ctx, request) + if err != nil { + return result, err + } + for _, reclaimed := range result.Reclaimed { + r.hooks.CompleteIssueOperationReclaim(ctx, reclaimed.ID) + } + return result, nil +} diff --git a/internal/storage/hook_lease_reclaimer_test.go b/internal/storage/hook_lease_reclaimer_test.go new file mode 100644 index 000000000..a3d8506df --- /dev/null +++ b/internal/storage/hook_lease_reclaimer_test.go @@ -0,0 +1,83 @@ +package storage + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +type fakeLeaseReclaimer struct { + result issueops.ReclaimResult + err error +} + +func (f *fakeLeaseReclaimer) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + return f.result, f.err +} + +// TestHookLeaseReclaimerFiresOncePerReclaimedRow is the decorator's whole +// subject: one on_update completion for every row the sweep reverted, in the +// sweep's order, and none at all for a refusal — even a refusal that somehow +// carries rows, because the role leaves its result unspecified on error. +func TestHookLeaseReclaimerFiresOncePerReclaimedRow(t *testing.T) { + for _, test := range []struct { + name string + inner *fakeLeaseReclaimer + wantFire []string + }{ + { + name: "every reverted row fires once", + inner: &fakeLeaseReclaimer{result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{ + {ID: "bd-1"}, {ID: "bd-2"}, + }}}, + wantFire: []string{"reclaim:bd-1", "reclaim:bd-2"}, + }, + { + name: "a sweep that reverted nothing fires nothing", + inner: &fakeLeaseReclaimer{result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{}}}, + }, + { + name: "a refusal fires nothing", + inner: &fakeLeaseReclaimer{err: errors.New("boom")}, + }, + { + name: "a refusal that carries rows still fires nothing", + inner: &fakeLeaseReclaimer{ + result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{{ID: "bd-1"}}}, + err: errors.New("boom"), + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + recorder := &recordingIssueOperationHooks{} + reclaimer := &hookLeaseReclaimer{inner: test.inner, hooks: recorder} + + _, err := reclaimer.Reclaim(context.Background(), issueops.ReclaimRequest{Actor: "reaper"}) + if (err != nil) != (test.inner.err != nil) { + t.Fatalf("Reclaim error = %v, want %v", err, test.inner.err) + } + if !reflect.DeepEqual(recorder.completions, test.wantFire) { + t.Fatalf("hooks fired = %v, want %v", recorder.completions, test.wantFire) + } + }) + } +} + +// TestHookLeaseReclaimerPassesTheResultThrough pins that the decorator reports +// exactly what the role reported, revisions included: a hook layer that +// rewrote the result would be deciding which token a caller feeds forward. +func TestHookLeaseReclaimerPassesTheResultThrough(t *testing.T) { + want := issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{{ID: "bd-1", PreviousOwner: "w", Revision: "42"}}} + reclaimer := &hookLeaseReclaimer{inner: &fakeLeaseReclaimer{result: want}, hooks: &recordingIssueOperationHooks{}} + + got, err := reclaimer.Reclaim(context.Background(), issueops.ReclaimRequest{Actor: "reaper"}) + if err != nil { + t.Fatalf("Reclaim error = %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("Reclaim result = %+v, want %+v", got, want) + } +} diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 0838bce48..be4703360 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -66,6 +66,7 @@ go_library( "journal_prune.go", "labels.go", "lease.go", + "lease_reclaimer.go", "metadata.go", "metadata_cas.go", "molecule.go", @@ -164,6 +165,7 @@ go_test( "journal_truncation_test.go", "journal_vocabulary_test.go", "label_rename_merged_count_test.go", + "lease_reclaimer_test.go", "lease_replica_test.go", "metadata_test.go", "prepare_timestamps_utc_test.go", diff --git a/internal/storage/issueops/lease.go b/internal/storage/issueops/lease.go index d2b75f7e7..2ec940280 100644 --- a/internal/storage/issueops/lease.go +++ b/internal/storage/issueops/lease.go @@ -667,6 +667,11 @@ func reclaimReplicaSQL(filter types.ReclaimFilter, localNode string) (string, [] // the lease from) so the caller can log/emit recovery events. The caller owns // Dolt versioning. // +// This is the one place a reclaimed row's revision token is minted: the row's +// freshRowLock() value is captured off the revert UPDATE and published on +// each types.ReclaimedLease as types.RevisionToken(newLock), the same +// encoding IssueDetails.Revision uses. +// // filter narrows which stale leases are eligible (see types.ReclaimFilter); the // zero filter keeps the historical global behavior. Scoping is applied to the // snapshot SELECT only — the per-row DELETE/UPDATE re-checks staleness by id, @@ -754,12 +759,13 @@ func ReclaimExpiredLeasesInTx(ctx context.Context, tx DBTX, cutoff time.Time, fi // being in_progress under us (closed) is left alone; row_lock makes a // concurrent close/update conflict at commit time rather than // cell-merge with this write. + newLock := freshRowLock() res, err = tx.ExecContext(ctx, ` UPDATE issues SET status = 'open', assignee = NULL, started_at = NULL, updated_at = ?, row_lock = ? WHERE id = ? AND status = 'in_progress' - `, time.Now().UTC(), freshRowLock(), r.ID) + `, time.Now().UTC(), newLock, r.ID) if err != nil { return nil, fmt.Errorf("reclaim %s: %w", r.ID, err) } @@ -770,6 +776,7 @@ func ReclaimExpiredLeasesInTx(ctx context.Context, tx DBTX, cutoff time.Time, fi if n == 0 { continue // no longer in_progress — its lease row was stale anyway } + r.Revision = types.RevisionToken(newLock) if err := RecordFullEventInTable(ctx, tx, "events", r.ID, types.EventLeaseReclaimed, actor, r.PreviousOwner, ""); err != nil { return nil, fmt.Errorf("record reclaim event for %s: %w", r.ID, err) diff --git a/internal/storage/issueops/lease_reclaimer.go b/internal/storage/issueops/lease_reclaimer.go new file mode 100644 index 000000000..ba0e8b4cb --- /dev/null +++ b/internal/storage/issueops/lease_reclaimer.go @@ -0,0 +1,123 @@ +package issueops + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// ValidateReclaimRequest applies the request rules every LeaseReclaimer +// implementation shares. +// +// THE CAP IS CHECKED AGAINST Filter.IDs AS SENT, before deduplication, for +// ValidateGetManyRequest's reason: the obligation it bounds is reading the +// request apart, not reading the rows it resolves to. +// +// THE ACTOR IS BOUNDED HERE, trimmed as the sweep records it, for +// claimIssueInTx's reason: every reverted row's recovery event lands in a +// VARCHAR(255) column, so an over-long actor is a typed refusal on every leg +// rather than a raw backend error on the first sweep that reverts something. +// +// It is the ONE definition of the role's rules: the library legs run it, and +// the HTTP client restates it (pinned by +// TestReclaimValidationMatchesTheSharedValidator). Every refusal but the cap +// is a *publicops.ReclaimFieldError naming its field. bd serve's handler leaves +// the id cap and the blank entries to it, and adds three rules of the wire's +// own that a library caller is not held to: the claim's name rules on the +// actor (a 256-byte bound, no control characters), an `older_than_seconds` a +// time.Duration can hold, and no scope member present but empty. +func ValidateReclaimRequest(request publicops.ReclaimRequest) error { + actor := strings.TrimSpace(request.Actor) + if actor == "" { + return reclaimFieldError(publicops.ReclaimFieldActor, "reclaim actor is required") + } + if err := types.CheckFieldLen("actor", actor); err != nil { + return reclaimFieldError(publicops.ReclaimFieldActor, err.Error()) + } + if request.OlderThan < 0 { + return reclaimFieldError(publicops.ReclaimFieldOlderThan, "reclaim older_than must not be negative") + } + if len(request.Filter.IDs) > publicops.MaxReclaimIDs { + return &publicops.TooManyReclaimIDsError{Requested: len(request.Filter.IDs), Cap: publicops.MaxReclaimIDs} + } + for i, id := range request.Filter.IDs { + if id == "" { + return reclaimFieldError(publicops.ReclaimFieldIDs, fmt.Sprintf("reclaim scope id at position %d is empty", i)) + } + } + for _, scope := range []struct { + field string + values []string + }{ + {publicops.ReclaimFieldAssignees, request.Filter.Assignees}, + {publicops.ReclaimFieldLabels, request.Filter.Labels}, + {publicops.ReclaimFieldLabelsAny, request.Filter.LabelsAny}, + {publicops.ReclaimFieldExcludeLabels, request.Filter.ExcludeLabels}, + } { + for i, v := range scope.values { + if strings.TrimSpace(v) == "" { + return reclaimFieldError(scope.field, fmt.Sprintf("reclaim %s entry at position %d is blank", scope.field, i)) + } + } + } + return nil +} + +func reclaimFieldError(field, detail string) error { + return &publicops.ReclaimFieldError{Field: field, Detail: storage.ErrValidation.Error() + ": " + detail} +} + +// ExecuteReclaimInTx sweeps stale leases from tx, in ONE transaction. It is +// the body behind the LeaseReclaimer accessor on ALL THREE legs: the two +// stores wrap it in their own retrying write transaction, and the +// unit-of-work provider reaches it through the domain repository, whose +// runner publishes exactly the DBTX method set ReclaimExpiredLeasesInTx +// takes. +// +// VALIDATION HAPPENS HERE, for ExecuteGetMany's reason: this body is the only +// body, so a leg that forgot to validate would be answering a different +// contract. It runs before the cutoff is computed and before the sweep's +// query, so a request that fails validation never reaches storage. +// +// THE CUTOFF IS COMPUTED HERE, once, from time.Now().UTC().Add(-OlderThan), so +// every leg sweeps against the same clock read rather than each leg taking its +// own — the dolt and embedded-dolt wrappers and the unit-of-work repository +// all call this function with the request, never with a precomputed cutoff. +func ExecuteReclaimInTx(ctx context.Context, tx DBTX, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { + if err := ValidateReclaimRequest(request); err != nil { + return publicops.ReclaimResult{}, err + } + cutoff := time.Now().UTC().Add(-request.OlderThan) + reclaimed, err := ReclaimExpiredLeasesInTx(ctx, tx, cutoff, request.Filter, strings.TrimSpace(request.Actor)) + if err != nil { + return publicops.ReclaimResult{}, fmt.Errorf("reclaim: %w", err) + } + if reclaimed == nil { + reclaimed = []publicops.ReclaimedLease{} + } + return publicops.ReclaimResult{Reclaimed: reclaimed}, nil +} + +// ReclaimVersionCommit composes the version-control entry for a finished sweep: +// the durable tables it changed and the message to record them under. A sweep +// that reverted nothing composes neither, which every leg's commit funnel reads +// as "write nothing to history", so a no-op reaper tick leaves no entry +// describing a no-op. +// +// It is ONE helper the three legs share so the message cannot drift between +// them: the same "bd: reclaim N expired lease(s)" the raw ReclaimExpiredLeases +// has always recorded. The staged set is the one the release records, issues +// and events, because a reclaim is the same revert applied by a clock. +func ReclaimVersionCommit(result publicops.ReclaimResult) (ChangedTables, string) { + if len(result.Reclaimed) == 0 { + return nil, "" + } + tables := ChangedTables{} + tables.Add("issues", "events") + return tables, fmt.Sprintf("bd: reclaim %d expired lease(s)", len(result.Reclaimed)) +} diff --git a/internal/storage/issueops/lease_reclaimer_test.go b/internal/storage/issueops/lease_reclaimer_test.go new file mode 100644 index 000000000..5b97b50af --- /dev/null +++ b/internal/storage/issueops/lease_reclaimer_test.go @@ -0,0 +1,107 @@ +package issueops + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestValidateReclaimRequestRefusesInTheDocumentedOrder pins the database-free +// tier of the role's refusals: each rule on both sides, and the cap checked +// before the blank scan, against the request as sent. +func TestValidateReclaimRequestRefusesInTheDocumentedOrder(t *testing.T) { + overCap := make([]string, publicops.MaxReclaimIDs+1) + atCap := make([]string, publicops.MaxReclaimIDs) + for i := range atCap { + atCap[i] = "bd-x" + } + for _, test := range []struct { + name string + request publicops.ReclaimRequest + wantErr bool + wantCap bool + }{ + {"a bare valid request", publicops.ReclaimRequest{Actor: "reaper"}, false, false}, + {"a zero grace window", publicops.ReclaimRequest{Actor: "reaper", OlderThan: 0}, false, false}, + {"exactly the cap", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{IDs: atCap}}, false, false}, + {"no actor", publicops.ReclaimRequest{}, true, false}, + {"a blank actor", publicops.ReclaimRequest{Actor: " "}, true, false}, + // The actor is bounded as the sweep records it: trimmed, then held to + // its column, so padding around a value at the bound is not over it. + {"an actor at the bound", publicops.ReclaimRequest{Actor: strings.Repeat("r", types.MaxFieldLen)}, false, false}, + {"a padded actor at the bound", publicops.ReclaimRequest{Actor: " " + strings.Repeat("r", types.MaxFieldLen) + " "}, false, false}, + {"an over-long actor", publicops.ReclaimRequest{Actor: strings.Repeat("r", types.MaxFieldLen+1)}, true, false}, + {"a negative grace window", publicops.ReclaimRequest{Actor: "reaper", OlderThan: -time.Nanosecond}, true, false}, + {"a blank id", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{IDs: []string{"bd-1", ""}}}, true, false}, + {"a blank assignee", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{Assignees: []string{"w", " "}}}, true, false}, + {"a blank label", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{Labels: []string{""}}}, true, false}, + {"a blank label-any", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{LabelsAny: []string{"\t"}}}, true, false}, + {"a blank exclude", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{ExcludeLabels: []string{"x", ""}}}, true, false}, + // Every entry blank AND one past the cap: the cap answers, so a caller + // that sent 1001 ids learns about the cap rather than about an entry. + {"over the cap of blanks", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{IDs: overCap}}, true, true}, + } { + t.Run(test.name, func(t *testing.T) { + err := ValidateReclaimRequest(test.request) + if (err != nil) != test.wantErr { + t.Fatalf("ValidateReclaimRequest = %v, want error %v", err, test.wantErr) + } + if err == nil { + return + } + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("ValidateReclaimRequest = %v, want ErrValidation", err) + } + var capErr *publicops.TooManyReclaimIDsError + if got := errors.As(err, &capErr); got != test.wantCap { + t.Fatalf("errors.As(*TooManyReclaimIDsError) = %v, want %v (err %v)", got, test.wantCap, err) + } + var fieldErr *publicops.ReclaimFieldError + if !test.wantCap && (!errors.As(err, &fieldErr) || fieldErr.Field == "") { + t.Fatalf("refusal %v names no field; every refusal but the cap is a *ReclaimFieldError", err) + } + }) + } +} + +// TestExecuteReclaimInTxRefusesBeforeAnyQuery pins "a request that fails +// validation never reaches storage": the mock expects nothing, so any query +// would fail the test. +func TestExecuteReclaimInTxRefusesBeforeAnyQuery(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + _, err = ExecuteReclaimInTx(context.Background(), db, publicops.ReclaimRequest{}) + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("ExecuteReclaimInTx with no actor = %v, want ErrValidation", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +// TestReclaimVersionCommitComposesNothingForANoOp pins the shared commit +// composition: no entry for a sweep that reverted nothing, and one entry +// naming the count, staging issues and events, for one that did. +func TestReclaimVersionCommitComposesNothingForANoOp(t *testing.T) { + if tables, msg := ReclaimVersionCommit(publicops.ReclaimResult{Reclaimed: []publicops.ReclaimedLease{}}); len(tables) != 0 || msg != "" { + t.Fatalf("a no-op sweep composed tables=%v msg=%q, want neither", tables, msg) + } + tables, msg := ReclaimVersionCommit(publicops.ReclaimResult{Reclaimed: []publicops.ReclaimedLease{{ID: "a"}, {ID: "b"}}}) + if !tables["issues"] || !tables["events"] || len(tables) != 2 { + t.Fatalf("tables = %v, want issues and events", tables) + } + if !strings.Contains(msg, "reclaim 2 expired lease(s)") { + t.Fatalf("message = %q, want it to name the two reverted leases", msg) + } +} diff --git a/internal/storage/role_accessor_decorator_test.go b/internal/storage/role_accessor_decorator_test.go index aa8ce3b57..540518354 100644 --- a/internal/storage/role_accessor_decorator_test.go +++ b/internal/storage/role_accessor_decorator_test.go @@ -68,11 +68,11 @@ func roleAccessorNamesOf(surface reflect.Type) (names, unclassified []string) { } // TestEveryStoreRoleAccessorIsClassified fails when DoltStorage hands out an -// interface the census cannot place. Every one of the twenty-eight today is a +// interface the census cannot place. Every one of the twenty-nine today is a // facade role, so this costs nothing and closes the path where a role surface // grows a package and the census quietly stops covering it. (roleAccessorNames // is derived, so the count is prose and only this sentence goes stale — but it -// does go stale, and the last three sweeps found it saying twenty-seven.) +// does go stale, and the last three sweeps found it saying twenty-eight.) func TestEveryStoreRoleAccessorIsClassified(t *testing.T) { _, unclassified := roleAccessorNamesOf(reflect.TypeOf((*DoltStorage)(nil)).Elem()) for _, accessor := range unclassified { @@ -151,71 +151,73 @@ func assertRoleAccessorsAreDeclared(t *testing.T, decorator reflect.Type) { } // roleAccessorStore is a DoltStorage whose only real methods are the -// twenty-eight role accessors, each answering with a distinguishable sentinel +// twenty-nine role accessors, each answering with a distinguishable sentinel // so a test can tell a decorated surface from a passed-through one. type roleAccessorStore struct { DoltStorage - lifecycle issueops.Lifecycle - reader issueops.Reader - relations issueops.Relations - edges issueops.EdgeReader - blocking issueops.BlockingAnnotator - tree issueops.TreeWalker - graphCounter issueops.GraphCounter - batchGetter issueops.BatchGetter - counter issueops.Counter - settings issueops.WorkspaceConfig - memories memoryops.Memories - versions issueops.VersionReconciler - stats issueops.StatsReporter - cycles issueops.CycleDetector - commenter issueops.Commenter - claimer issueops.ReadyClaimer - closer issueops.BatchCloser - creator issueops.BatchCreator - editor issueops.DependencyEditor - applier issueops.BatchApplier - readyCounter issueops.ReadyCounter - querier issueops.Querier - sweeper issueops.Sweeper - deleter issueops.Deleter - bootstrapper issueops.Bootstrapper - verifier issueops.InitVerifier - metadataCAS issueops.MetadataCAS - releaser issueops.Releaser - err error + lifecycle issueops.Lifecycle + reader issueops.Reader + relations issueops.Relations + edges issueops.EdgeReader + blocking issueops.BlockingAnnotator + tree issueops.TreeWalker + graphCounter issueops.GraphCounter + batchGetter issueops.BatchGetter + counter issueops.Counter + settings issueops.WorkspaceConfig + memories memoryops.Memories + versions issueops.VersionReconciler + stats issueops.StatsReporter + cycles issueops.CycleDetector + commenter issueops.Commenter + claimer issueops.ReadyClaimer + closer issueops.BatchCloser + creator issueops.BatchCreator + editor issueops.DependencyEditor + applier issueops.BatchApplier + readyCounter issueops.ReadyCounter + querier issueops.Querier + sweeper issueops.Sweeper + deleter issueops.Deleter + bootstrapper issueops.Bootstrapper + verifier issueops.InitVerifier + metadataCAS issueops.MetadataCAS + releaser issueops.Releaser + leaseReclaimer issueops.LeaseReclaimer + err error } func newRoleAccessorStore() *roleAccessorStore { sentinel := &roleAccessorSentinel{} return &roleAccessorStore{ - memories: &memoryRoleSentinel{}, - lifecycle: sentinel, - reader: sentinel, - relations: sentinel, - edges: sentinel, - blocking: sentinel, - graphCounter: sentinel, - batchGetter: sentinel, - counter: sentinel, - settings: sentinel, - versions: sentinel, - stats: sentinel, - cycles: sentinel, - commenter: sentinel, - claimer: sentinel, - closer: sentinel, - creator: sentinel, - editor: sentinel, - applier: sentinel, - readyCounter: sentinel, - querier: sentinel, - sweeper: sentinel, - deleter: sentinel, - bootstrapper: sentinel, - verifier: sentinel, - metadataCAS: sentinel, - releaser: sentinel, + memories: &memoryRoleSentinel{}, + lifecycle: sentinel, + reader: sentinel, + relations: sentinel, + edges: sentinel, + blocking: sentinel, + graphCounter: sentinel, + batchGetter: sentinel, + counter: sentinel, + settings: sentinel, + versions: sentinel, + stats: sentinel, + cycles: sentinel, + commenter: sentinel, + claimer: sentinel, + closer: sentinel, + creator: sentinel, + editor: sentinel, + applier: sentinel, + readyCounter: sentinel, + querier: sentinel, + sweeper: sentinel, + deleter: sentinel, + bootstrapper: sentinel, + verifier: sentinel, + metadataCAS: sentinel, + releaser: sentinel, + leaseReclaimer: sentinel, } } @@ -286,8 +288,11 @@ func (s *roleAccessorStore) BatchApplier() (issueops.BatchApplier, error) { func (s *roleAccessorStore) Releaser() (issueops.Releaser, error) { return s.releaser, s.err } +func (s *roleAccessorStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return s.leaseReclaimer, s.err +} -// roleAccessorSentinel implements twenty-seven of the twenty-eight roles at +// roleAccessorSentinel implements twenty-eight of the twenty-nine roles at // once — every one but memoryops.Memories, whose List collides with // issueops.Reader.List and needs the second sentinel below. // Nothing calls its methods; identity is the whole point. @@ -412,6 +417,9 @@ func (*roleAccessorSentinel) CompareAndSetKey(context.Context, issueops.CompareA func (*roleAccessorSentinel) Release(context.Context, issueops.ReleaseRequest) (issueops.ReleaseResult, error) { return issueops.ReleaseResult{}, nil } +func (*roleAccessorSentinel) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + return issueops.ReclaimResult{}, nil +} // memoryRoleSentinel is the memory role's sentinel, and the one role // memoryRoleSentinel is the remaining role's sentinel, and the one role @@ -474,6 +482,7 @@ func TestHookFiringStoreWrapsTheWriteRolesAndPassesTheReadsThrough(t *testing.T) {"MetadataCAS", func() (any, error) { return store.MetadataCAS() }, inner.metadataCAS, true}, {"BatchApplier", func() (any, error) { return store.BatchApplier() }, inner.applier, true}, {"Releaser", func() (any, error) { return store.Releaser() }, inner.releaser, true}, + {"LeaseReclaimer", func() (any, error) { return store.LeaseReclaimer() }, inner.leaseReclaimer, true}, {"IssueReader", func() (any, error) { return store.IssueReader() }, inner.reader, false}, {"IssueRelations", func() (any, error) { return store.IssueRelations() }, inner.relations, false}, {"EdgeReader", func() (any, error) { return store.EdgeReader() }, inner.edges, false}, @@ -532,6 +541,7 @@ func TestHookFiringStoreRoleAccessorsPropagateInnerErrors(t *testing.T) { {"MetadataCAS", func() (any, error) { return store.MetadataCAS() }}, {"BatchApplier", func() (any, error) { return store.BatchApplier() }}, {"Releaser", func() (any, error) { return store.Releaser() }}, + {"LeaseReclaimer", func() (any, error) { return store.LeaseReclaimer() }}, {"IssueReader", func() (any, error) { return store.IssueReader() }}, {"IssueRelations", func() (any, error) { return store.IssueRelations() }}, {"EdgeReader", func() (any, error) { return store.EdgeReader() }}, diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 8abe0b0fd..66553307f 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -378,6 +378,15 @@ type Storage interface { // assignee and status, which is on_update — the same event the journal // already records for it. See hook_releaser.go. Releaser() (issueops.Releaser, error) + // LeaseReclaimer returns the lease-sweep surface for this store: revert + // every lease that has been held past its deadline back to its + // pre-claim state, for a caller that is a clock rather than a human or a + // request. + // + // It is a WRITE role and its hook decorator WRAPS: a reclaim changes + // assignee and status per reverted row, which is on_update, the same + // event Releaser fires. See hook_lease_reclaimer.go. + LeaseReclaimer() (issueops.LeaseReclaimer, error) // Issue CRUD CreateIssue(ctx context.Context, issue *types.Issue, actor string) error diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 8b4abc0d4..fed8ccd9a 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -31,6 +31,7 @@ go_library( "issue_operations.go", "issue_reader.go", "journal_maintenance.go", + "lease_reclaimer.go", "maintenance.go", "memories.go", "metadata_cas.go", @@ -128,6 +129,7 @@ go_test( "journal_contract_test.go", "journal_delete_actor_test.go", "journal_scope_test.go", + "lease_reclaimer_contract_test.go", "lifecycle_close_reopen_contract_test.go", "lifecycle_create_contract_test.go", "lifecycle_role_hooks_test.go", diff --git a/internal/storage/uow/lease_reclaimer.go b/internal/storage/uow/lease_reclaimer.go new file mode 100644 index 000000000..1cc522c6b --- /dev/null +++ b/internal/storage/uow/lease_reclaimer.go @@ -0,0 +1,60 @@ +package uow + +import ( + "context" + "fmt" + + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + publicops "github.com/steveyegge/beads/issueops" +) + +// LeaseReclaimerSource is the capability accessor a unit-of-work provider +// offers for the lease-sweep role, the sibling of DeleterSource and +// ReleaserSource. +type LeaseReclaimerSource interface { + LeaseReclaimer() (publicops.LeaseReclaimer, error) +} + +// leaseReclaimer sweeps stale leases through a unit of work. +type leaseReclaimer struct { + provider UnitOfWorkProvider +} + +// LeaseReclaimer returns the guarded lease-sweep surface for this provider. +func (p *doltSQLProvider) LeaseReclaimer() (publicops.LeaseReclaimer, error) { + return NewLeaseReclaimer(p) +} + +// NewLeaseReclaimer constructs a public LeaseReclaimer backed by provider. +func NewLeaseReclaimer(provider UnitOfWorkProvider) (publicops.LeaseReclaimer, error) { + if isNilUnitOfWorkProvider(provider) { + return nil, fmt.Errorf("new lease reclaimer: unit-of-work provider must not be nil") + } + return &leaseReclaimer{provider: provider}, nil +} + +var _ publicops.LeaseReclaimer = (*leaseReclaimer)(nil) + +// Reclaim reaches domain.IssueUseCase.Reclaim inside ONE committing unit of +// work, composing the same conditional commit message the dolt and +// embedded-dolt legs use: a sweep that reverts nothing leaves no history +// entry describing a no-op. +// +// VALIDATION HAPPENS BEFORE THE UNIT OF WORK OPENS, as on those two legs and +// for Releaser's reason: a refusal then changes nothing on the connection as +// well as on the rows. The shared body validates again inside the transaction, +// which is what keeps a leg that skipped this from answering a different +// contract. +func (l *leaseReclaimer) Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { + if err := storageissueops.ValidateReclaimRequest(request); err != nil { + return publicops.ReclaimResult{}, err + } + return RunTxResult(ctx, l.provider, func(ctx context.Context, uw UnitOfWork) (publicops.ReclaimResult, string, error) { + result, err := uw.IssueUseCase().Reclaim(ctx, request) + if err != nil { + return result, "", err + } + _, msg := storageissueops.ReclaimVersionCommit(result) + return result, msg, nil + }) +} diff --git a/internal/storage/uow/lease_reclaimer_contract_test.go b/internal/storage/uow/lease_reclaimer_contract_test.go new file mode 100644 index 000000000..edcda454a --- /dev/null +++ b/internal/storage/uow/lease_reclaimer_contract_test.go @@ -0,0 +1,146 @@ +package uow + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestLeaseReclaimerContract runs the LeaseReclaimer contract against the +// unit-of-work provider, which reaches internal/storage/issueops. +// ExecuteReclaimInTx through domain.IssueUseCase.Reclaim inside one committing +// unit of work. +func TestLeaseReclaimerContract(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + fixture := newUOWLeaseReclaimerFixture(t, ctx, "lrc") + + t.Run("RevertsAStaleLease", func(t *testing.T) { + conformance.RunLeaseReclaimerRevertsAStaleLease(t, ctx, fixture) + }) + t.Run("MintsARevisionThatFencesTheFormerHolder", func(t *testing.T) { + conformance.RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t, ctx, fixture) + }) + t.Run("LeavesALiveLeaseAlone", func(t *testing.T) { + conformance.RunLeaseReclaimerLeavesALiveLeaseAlone(t, ctx, fixture) + }) + t.Run("HonorsTheGraceWindow", func(t *testing.T) { + conformance.RunLeaseReclaimerHonorsTheGraceWindow(t, ctx, fixture) + }) + t.Run("ScopesToTheNamedIDs", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToTheNamedIDs(t, ctx, fixture) + }) + t.Run("AnswersAnAbsentIDWithAnEmptyResult", func(t *testing.T) { + conformance.RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t, ctx, fixture) + }) + t.Run("ScopesToAssigneesAndLabels", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToAssigneesAndLabels(t, ctx, fixture) + }) + t.Run("AttributesTheRecoveryEvent", func(t *testing.T) { + conformance.RunLeaseReclaimerAttributesTheRecoveryEvent(t, ctx, fixture) + }) + t.Run("FiresOnUpdateOncePerReclaimedRow", func(t *testing.T) { + conformance.RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t, ctx, fixture) + }) + t.Run("RefusesAMalformedRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesAMalformedRequest(t, ctx, fixture) + }) + t.Run("RefusesABlankScopeEntry", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesABlankScopeEntry(t, ctx, fixture) + }) + t.Run("AcceptsExactlyTheCap", func(t *testing.T) { + conformance.RunLeaseReclaimerAcceptsExactlyTheCap(t, ctx, fixture) + }) + t.Run("DoesNotMutateTheCallerRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerDoesNotMutateTheCallerRequest(t, ctx, fixture) + }) +} + +// TestLeaseReclaimerRefusesBeforeOpeningAUOW: a malformed sweep is a +// deterministic validation failure and must not cost a database connection to +// discover, as on the dolt and embedded-dolt legs. The provider fails every +// open, so a refusal that came from inside a unit of work would be counted. +func TestLeaseReclaimerRefusesBeforeOpeningAUOW(t *testing.T) { + for _, tc := range []struct { + name string + request publicops.ReclaimRequest + }{ + {"no actor", publicops.ReclaimRequest{}}, + {"a negative grace window", publicops.ReclaimRequest{Actor: "reaper", OlderThan: -1}}, + {"a blank id", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{IDs: []string{""}}}}, + } { + t.Run(tc.name, func(t *testing.T) { + provider := &mockUnitOfWorkProvider{newUOWErr: errors.New("unexpected unit-of-work open")} + reclaimer, err := NewLeaseReclaimer(provider) + if err != nil { + t.Fatalf("NewLeaseReclaimer: %v", err) + } + if _, err := reclaimer.Reclaim(context.Background(), tc.request); !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("err = %v, want ErrValidation", err) + } + if provider.newUOWCalls != 0 { + t.Errorf("an invalid request opened %d units of work", provider.newUOWCalls) + } + }) + } +} + +// newUOWLeaseReclaimerFixture takes every role through the provider's own +// capability accessors, and the hooked reclaimer off the notifying provider +// the proxied CLI route wraps around its provider. +func newUOWLeaseReclaimerFixture(t *testing.T, ctx context.Context, prefix string) conformance.LeaseReclaimerFixture { + t.Helper() + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + source, ok := provider.(LeaseReclaimerSource) + if !ok { + t.Fatalf("provider %T does not offer the LeaseReclaimer accessor", provider) + } + reclaimer, err := source.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + claimer, err := provider.(IssueClaimerSource).IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + lifecycle, err := provider.(IssueLifecycleSource).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := reclaimer + if runner != nil { + notifying, ok := NewNotifyingProvider(provider, Sinks{Hook: runner}).(LeaseReclaimerSource) + if !ok { + t.Fatal("the notifying provider does not offer the LeaseReclaimer accessor") + } + if hooked, err = notifying.LeaseReclaimer(); err != nil { + t.Fatalf("hooked LeaseReclaimer(): %v", err) + } + } + kit := newUOWRoleFixtureKit(provider, prefix) + return conformance.LeaseReclaimerFixture{ + IssuePrefix: kit.IssuePrefix, + LeaseReclaimer: reclaimer, + CreateIssue: kit.CreateIssue, + Claimer: claimer, + Lifecycle: lifecycle, + QueryScalar: kit.QueryScalar, + Exec: func(ctx context.Context, statements []conformance.SQLStatement) error { + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + for _, stmt := range statements { + if _, err := uw.RawSQLUseCase().Exec(ctx, stmt.Query, stmt.Args...); err != nil { + return "", fmt.Errorf("%s: %w", stmt.Query, err) + } + } + return "seed lease reclaimer rows", nil + }) + }, + HookedLeaseReclaimer: hooked, + FiredUpdates: fired, + } +} diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index f1e9930e5..53da8a3b9 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -309,6 +309,10 @@ func (p *notifyingProvider) MetadataCAS() (publicops.MetadataCAS, error) { retur func (p *notifyingProvider) Releaser() (publicops.Releaser, error) { return NewReleaser(p) } +func (p *notifyingProvider) LeaseReclaimer() (publicops.LeaseReclaimer, error) { + return NewLeaseReclaimer(p) +} + func (p *notifyingProvider) Memories() (memoryops.Memories, error) { return NewMemories(p) } // EventsJournalCursor builds on THIS provider, like every role above it, so a @@ -423,6 +427,7 @@ var ( _ WorkspaceConfigSource = (*notifyingProvider)(nil) _ VersionReconcilerSource = (*notifyingProvider)(nil) _ MemoriesSource = (*notifyingProvider)(nil) + _ LeaseReclaimerSource = (*notifyingProvider)(nil) _ EventsJournalCursorSource = (*notifyingProvider)(nil) ) @@ -1008,6 +1013,25 @@ func (u *recordingIssueUC) ReleaseIssue(ctx context.Context, req publicops.Relea return result, wrote, nil } +// Reclaim records an update for every lease the sweep reverted. +// +// It is DECLARED for the same reason ReleaseIssue above is: an inherited +// accessor would compile and record nothing, silently dropping the hook the +// DoltStorage chain fires once per reverted row. +// +// A sweep reverts only issues, never wisps, so the snapshot is the issues- +// plane snap.issue rather than anyPlane. +func (u *recordingIssueUC) Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { + result, err := u.IssueUseCase.Reclaim(ctx, request) + if err != nil { + return result, err + } + for _, r := range result.Reclaimed { + u.rec.record(opUpdate, u.snap.issue(ctx, r.ID)) + } + return result, nil +} + func (u *recordingIssueUC) UpdateWisp(ctx context.Context, id string, updates map[string]any, actor string) error { if err := u.IssueUseCase.UpdateWisp(ctx, id, updates, actor); err != nil { return err diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel index 90792516b..ffcf6b7d5 100644 --- a/internal/telemetry/BUILD.bazel +++ b/internal/telemetry/BUILD.bazel @@ -20,6 +20,7 @@ go_library( "issue_claimer.go", "issue_operations.go", "issue_reader.go", + "lease_reclaimer.go", "memories.go", "metadata_cas.go", "querier.go", diff --git a/internal/telemetry/lease_reclaimer.go b/internal/telemetry/lease_reclaimer.go new file mode 100644 index 000000000..cf61d8356 --- /dev/null +++ b/internal/telemetry/lease_reclaimer.go @@ -0,0 +1,36 @@ +package telemetry + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// LeaseReclaimer returns the inner store's lease-sweep surface wrapped in +// this layer's instrumentation. It recurses instead of delegating: a blind +// delegation would return the inner surface unspanned and untimed. +func (s *InstrumentedStorage) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + inner, err := s.Unwrap().LeaseReclaimer() + if err != nil { + return nil, err + } + return s.WrapLeaseReclaimer(inner), nil +} + +// WrapLeaseReclaimer instruments the lease sweep with this storage layer's +// existing telemetry meter and tracer. +func (s *InstrumentedStorage) WrapLeaseReclaimer(inner issueops.LeaseReclaimer) issueops.LeaseReclaimer { + return &instrumentedLeaseReclaimer{storage: s, inner: inner} +} + +type instrumentedLeaseReclaimer struct { + storage *InstrumentedStorage + inner issueops.LeaseReclaimer +} + +func (c *instrumentedLeaseReclaimer) Reclaim(ctx context.Context, request issueops.ReclaimRequest) (result issueops.ReclaimResult, err error) { + ctx, span, started := c.storage.op(ctx, "LeaseReclaimer.Reclaim") + result, err = c.inner.Reclaim(ctx, request) + c.storage.done(ctx, span, started, err) + return result, err +} diff --git a/internal/telemetry/role_accessor_decorator_test.go b/internal/telemetry/role_accessor_decorator_test.go index e7c9b7c9c..24a2d953b 100644 --- a/internal/telemetry/role_accessor_decorator_test.go +++ b/internal/telemetry/role_accessor_decorator_test.go @@ -94,7 +94,7 @@ func TestInstrumentedStorageDeclaresEveryRoleAccessor(t *testing.T) { } // roleAccessorStore is a DoltStorage whose only real methods are the -// twenty-eight role accessors, each answering with a distinguishable sentinel +// twenty-nine role accessors, each answering with a distinguishable sentinel // so a test can tell an instrumented surface from a passed-through one. // // TWO sentinels rather than one: memoryops.Memories.List and issueops.Reader.List @@ -175,8 +175,11 @@ func (s *roleAccessorStore) BatchApplier() (issueops.BatchApplier, error) { func (s *roleAccessorStore) Releaser() (issueops.Releaser, error) { return s.surface, s.err } +func (s *roleAccessorStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return s.surface, s.err +} -// roleAccessorSentinel implements twenty-seven of the twenty-eight roles at +// roleAccessorSentinel implements twenty-eight of the twenty-nine roles at // once — every one but memoryops.Memories, whose List collides with // issueops.Reader.List and needs the second sentinel below. // Nothing calls its methods; identity is the whole point. @@ -306,6 +309,9 @@ func (*roleAccessorSentinel) CompareAndSetKey(context.Context, issueops.CompareA func (*roleAccessorSentinel) Release(context.Context, issueops.ReleaseRequest) (issueops.ReleaseResult, error) { return issueops.ReleaseResult{}, nil } +func (*roleAccessorSentinel) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + return issueops.ReclaimResult{}, nil +} // memoryRoleSentinel is the memory role's sentinel — see // memoryRoleSentinel is the remaining role's sentinel — see @@ -382,6 +388,7 @@ func TestInstrumentedStorageInstrumentsEveryRoleAccessor(t *testing.T) { {"MetadataCAS", func() (any, error) { return wrapped.MetadataCAS() }, sentinel}, {"BatchApplier", func() (any, error) { return wrapped.BatchApplier() }, sentinel}, {"Releaser", func() (any, error) { return wrapped.Releaser() }, sentinel}, + {"LeaseReclaimer", func() (any, error) { return wrapped.LeaseReclaimer() }, sentinel}, } { t.Run(test.name, func(t *testing.T) { surface, err := test.got() @@ -468,6 +475,7 @@ func TestInstrumentedStorageRoleAccessorsPropagateInnerErrors(t *testing.T) { {"MetadataCAS", func() (any, error) { return wrapped.MetadataCAS() }}, {"BatchApplier", func() (any, error) { return wrapped.BatchApplier() }}, {"Releaser", func() (any, error) { return wrapped.Releaser() }}, + {"LeaseReclaimer", func() (any, error) { return wrapped.LeaseReclaimer() }}, } { t.Run(test.name, func(t *testing.T) { surface, err := test.got() diff --git a/internal/types/types.go b/internal/types/types.go index 12f3777e0..dffc85811 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -2380,6 +2380,13 @@ func (s SortPolicy) IsValid() bool { type ReclaimedLease struct { ID string `json:"id"` PreviousOwner string `json:"previous_owner"` + + // Revision is the fresh RevisionToken minted for this issue's row by the + // same UPDATE that reverted its lease — the one this package's + // RevisionToken/ParseRevisionToken pair encodes, the same wire shape as + // IssueDetails.Revision. A caller that re-fetches after a reclaim sees a + // different token than one that raced it. + Revision string `json:"revision"` } // ReclaimFilter scopes which stale-lease issues bd reclaim may revert. The diff --git a/issue_roles_external_test.go b/issue_roles_external_test.go index b6e6dd38a..c9fde9947 100644 --- a/issue_roles_external_test.go +++ b/issue_roles_external_test.go @@ -238,6 +238,39 @@ func TestReleaserExposesTypedUnsupportedError(t *testing.T) { } } +// TestLeaseReclaimerLayersHooksOutsideTelemetry is the same pin for the +// lease-sweep role. It gives the same answer the release does and for the +// same reason: a reclaim moves assignee and status per reverted row, which +// is on_update — a hook the vocabulary publishes and a row a script can +// still read back — so this decorator wraps rather than recursing the way +// Sweeper and Deleter do. +func TestLeaseReclaimerLayersHooksOutsideTelemetry(t *testing.T) { + t.Setenv("BD_OTEL_STDOUT", "true") + instrumented, ok := telemetry.WrapStorage(&dolt.DoltStore{}).(*telemetry.InstrumentedStorage) + if !ok { + t.Fatal("WrapStorage() did not create InstrumentedStorage") + } + + reclaimer, err := storage.NewHookFiringStore(instrumented, nil).LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer() error = %v", err) + } + if got := reflect.TypeOf(reclaimer).String(); got != "*storage.hookLeaseReclaimer" { + t.Fatalf("outer layer = %s, want the hook wrapper", got) + } +} + +func TestLeaseReclaimerExposesTypedUnsupportedError(t *testing.T) { + reclaimer, err := (*dolt.DoltStore)(nil).LeaseReclaimer() + if reclaimer != nil { + t.Fatalf("LeaseReclaimer() reclaimer = %T, want nil", reclaimer) + } + var unsupported *beads.ErrUnsupported + if !errors.As(err, &unsupported) { + t.Fatalf("LeaseReclaimer() error = %v, want *beads.ErrUnsupported", err) + } +} + // TestIssueRelationsKeepsTelemetryOutermost is the READ role's version of the // pin, and it is deliberately the other answer: the hook decorator adds no // layer to a read, so the outermost thing a caller gets is the instrumented diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index dc28cd166..47fa9a549 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -22,6 +22,7 @@ go_library( "importer.go", "initverifier.go", "issueops.go", + "leasereclaimer.go", "metadatacas.go", "querier.go", "reader.go", diff --git a/issueops/leasereclaimer.go b/issueops/leasereclaimer.go new file mode 100644 index 000000000..4c63f5a42 --- /dev/null +++ b/issueops/leasereclaimer.go @@ -0,0 +1,183 @@ +package issueops + +import ( + "context" + "fmt" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// ReclaimFilter scopes which stale-lease issues a reclaim may revert. See +// types.ReclaimFilter: this is the same value, aliased into this package the +// way GetManyRequest's sibling roles alias their types.* shapes. +type ReclaimFilter = types.ReclaimFilter + +// ReclaimedLease names one issue whose stale lease a reclaim reverted, plus +// the owner it took the lease from and the fresh revision the revert minted. +// See types.ReclaimedLease. +type ReclaimedLease = types.ReclaimedLease + +// MaxReclaimIDs is the largest ReclaimFilter.IDs this role accepts in one +// call. It is the same number and the same reasoning as MaxGetManyIDs: a flat +// bound on the request rather than a cost model over the sweep it scopes, so +// one call cannot hold the revert transaction open over an unbounded id list. +const MaxReclaimIDs = 1000 + +// TooManyReclaimIDsError reports that a ReclaimRequest's Filter.IDs named more +// ids than this role accepts in one call. It wraps ErrValidation, matching +// TooManyIDsError's shape for the same reason: a caller that only checks for a +// usable request classifies it correctly without knowing this type exists. +type TooManyReclaimIDsError struct { + // Requested is the number of ids Filter.IDs named, counted before + // deduplication. + Requested int + // Cap is MaxReclaimIDs, carried on the error so a caller does not need the + // constant in scope to report what it hit. + Cap int +} + +func (e *TooManyReclaimIDsError) Error() string { + return fmt.Sprintf("reclaim accepts at most %d scoped ids, got %d", e.Cap, e.Requested) +} + +// Unwrap makes TooManyReclaimIDsError match ErrValidation. +func (e *TooManyReclaimIDsError) Unwrap() error { return ErrValidation } + +// The ReclaimRequest fields a ReclaimFieldError can name. +const ( + ReclaimFieldActor = "actor" + ReclaimFieldOlderThan = "older_than" + ReclaimFieldIDs = "ids" + ReclaimFieldAssignees = "assignees" + ReclaimFieldLabels = "labels" + ReclaimFieldLabelsAny = "labels_any" + ReclaimFieldExcludeLabels = "exclude_labels" +) + +// ReclaimFieldError reports a ReclaimRequest refusal and the one field it is +// about, so a front door can name that field without parsing the message. It +// wraps ErrValidation. The id cap is the one refusal that is not one of these: +// it is a *TooManyReclaimIDsError, always about ReclaimFieldIDs. +type ReclaimFieldError struct { + // Field is one of the ReclaimField* constants. + Field string + // Detail is the sentence the error reads as. + Detail string +} + +func (e *ReclaimFieldError) Error() string { return e.Detail } + +// Unwrap makes ReclaimFieldError match ErrValidation. +func (e *ReclaimFieldError) Unwrap() error { return ErrValidation } + +// ReclaimRequest describes one sweep of stale leases — the shape behind +// `bd reclaim`. +// +// Implementations never mutate caller-owned request values: Filter's slices +// are read, never written through. +type ReclaimRequest struct { + // Actor attributes every reverted row's recovery event and its journaled + // update. It is REQUIRED — an empty or all-blank Actor is ErrValidation — + // for the same reason Releaser.Release requires one: a reclaim is the + // moment work stops being owned, and the one question asked of a reverted + // row's history afterwards is who ran the sweep that freed it. It is + // recorded trimmed, and one longer than types.MaxFieldLen characters once + // trimmed is ErrValidation too: the column it is recorded in holds no more. + Actor string + // OlderThan is the grace period past a lease's own TTL: only a lease whose + // lease_expires_at is more than OlderThan in the past is eligible. It must + // not be negative — a negative grace reaches into the future and reclaims + // leases that have not yet expired by their own TTL, which is not a wider + // sweep, it is a different operation this role does not perform. + OlderThan time.Duration + // Filter narrows which stale-lease issues are eligible; the zero value + // reclaims every stale lease workspace-wide. See types.ReclaimFilter for + // the scope fields (ids, assignees, labels) and the AnyReplica escape + // hatch — none of that is restated here, and this role changes none of it. + // + // FILTER.IDS IS NOT AN ERROR WHEN IT NAMES A NON-STALE OR UNKNOWN ISSUE. + // Like GetManyRequest.IDs, an id that is not currently a stale lease + // simply does not appear in ReclaimResult.Reclaimed — scoping by id + // narrows a sweep, it does not assert that every named id was reclaimable. + // + // MORE THAN MaxReclaimIDs ENTRIES IN Filter.IDs IS A REFUSAL: a + // *TooManyReclaimIDsError, counted as sent, before deduplication. + Filter ReclaimFilter +} + +// ReclaimResult reports every issue a reclaim reverted, in the order the sweep +// found them. +type ReclaimResult struct { + // Reclaimed names every issue the sweep reverted, together with the owner + // each lease was taken from and the fresh revision the revert minted on + // each row. Never nil for a successful call, even when it is empty: a + // sweep that finds nothing stale answers with an empty, non-nil slice, the + // same promise GetManyResult.Issues makes. + Reclaimed []ReclaimedLease +} + +// LeaseReclaimer sweeps stale leases back to ready — the capability behind +// `bd reclaim` — and, like every other capability here, a role with its own +// accessor. +// +// IT IS A DIFFERENT QUESTION FROM Releaser. Releaser gives up ONE claim a +// caller names, on the claim's own say-so (Actor, or Force). This sweeps +// MANY leases that have gone stale by a clock, on nobody's say-so but the +// clock's — a reaper calls this, not the agent that held the work. A surface +// carrying both would let a caller that may only report its own abandoned +// work reach into every other holder's, which is the same split Releaser's +// own doc draws against Claimer. +// +// WRITES, AND ITS HOOK IS on_update, once per reverted row. A reclaim clears +// assignee and status on each row it reverts, which is an update and is +// already how the journal classifies it — see Releaser's identical +// reasoning. There is no on_reclaim to fire and inventing one is not this +// role's job. +// +// Reclaim only ever touches the permanent issues table: wisps are ephemeral +// and are never leased work (see internal/storage/issueops. +// ReclaimExpiredLeasesInTx). The replica guard ReclaimFilter.AnyReplica +// documents is unchanged by this role — it is enforced in the shared body +// every library leg calls through. +// +// EVERY LEG RUNS ONE BODY: dolt, embedded dolt and the unit-of-work provider +// wrap internal/storage/issueops.ExecuteReclaimInTx, and the HTTP client leg +// dials POST /v0/beads/issues:reclaim (capability issues.reclaim), behind which +// bd serve runs the same body. `bd reclaim` runs this role on both of its +// routes. Each leg records one version-control entry per sweep that reverted +// anything, and honors a deferred version commit on the context. +// +// Deterministic request-validation failures match ErrValidation. Result +// values are unspecified when error is non-nil, and every refusal below +// leaves persistent state unchanged. +type LeaseReclaimer interface { + // Reclaim sweeps every stale lease ReclaimRequest.Filter admits whose + // lease_expires_at is more than OlderThan in the past, and reports every + // issue it reverted. + // + // IT IS ONE TRANSACTION over the whole sweep: the snapshot that decides + // which leases are stale, every row's revert, and the revision each revert + // mints all see one write. A lease a concurrent heartbeat rescues between + // the snapshot and its row's revert is simply skipped — it never appears + // in Reclaimed, and is not an error. + // + // REFUSALS: + // + // - an empty or all-blank Actor, or one longer than types.MaxFieldLen + // characters once trimmed: ErrValidation, before anything is read; + // - a negative OlderThan: ErrValidation; + // - more than MaxReclaimIDs entries in Filter.IDs, or an empty-string + // entry among them: *TooManyReclaimIDsError or ErrValidation, + // respectively, before anything is read; + // - an empty or all-blank entry in Filter.Assignees, Labels, LabelsAny or + // ExcludeLabels: ErrValidation, because it would match nothing a + // caller meant to name. + // + // Every ErrValidation refusal but the cap is a *ReclaimFieldError naming + // its field. + // + // A sweep that finds no stale leases is not a refusal: it answers an empty, + // non-nil Reclaimed. + Reclaim(ctx context.Context, request ReclaimRequest) (ReclaimResult, error) +} From f21e3a80f7ed3b3b572b329e6f847971e78890ba Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 16:25:37 +0000 Subject: [PATCH 016/151] fix(issueops): templates are read-only on every update leg (#7427) * fix(issueops): enforce the template read-only update guard on every leg The refusal of an update to a template (validateIssueUpdatable) lived only in cmd/bd as a pre-read, so PATCH /v0/beads/issues/{id}, batch:apply update items and library callers of Lifecycle.Update / BatchApplier edited templates that `bd update` refused. It is now one role rule, storage issueops.AuthorizeTemplateUpdate, returning *issueops.TemplateReadOnlyError (matches ErrTemplateReadOnly; the message is the sentence bd always printed). It runs in the update's own transaction, after the compare-and-set preconditions (a stale guard still reports the mismatch), and no force flag waives it: - ExecuteUpdate (dolt, embedded) and the uow Update and batch update legs; - the batch apply metadata splice that finishes a create item stays unguarded on both bodies (a template the request creates is not modified); - httpapi maps it to 409 template_read_only on updateIssue and applyBatch (spec x-bd-codes and descriptions updated, apigen regenerated) and the client rebuilds the typed error. UpdateRequest.AllowTemplate stands the guard down for one request: the caller edits a template deliberately. bd label (add, remove, remove-prefix) and bd set-state set it, so they keep editing templates on every route, as on origin/main. Over HTTP it is the optional `allow_template` member of the PATCH body, advertised by the issues.update.allowTemplate behavior token, which also says the server enforces the guard. httpclient Lifecycle.Update decides from the cached handshake before the dial: against a server without the token (no guard, no member) it runs the same rule (AuthorizeTemplateUpdate) on a pre-read and never sends allow_template, so a new bd against an older bd serve still refuses template updates. cmd/bd: bd update, bd assign and the proxied assign/tag drop their pre-read and print the role's refusal with the same line and exit code. Conformance: RunLifecycleUpdateRefusesATemplate, RunLifecycleUpdateAllowTemplateEditsATemplate, RunBatchApplyUpdateItemsRefuseATemplate and RunBatchApplySplicesTheMetadataOfATemplateItCreates on dolt, embedded, uow and http (the http splice case is parked: the v0 create item has no is_template member), plus handler and CLI rendering tests, and TestUpdateTemplateGuardAgainstAServerThatPredatesIt (new client, handshake without the token: refused before the dial, allow_template not sent). Skew note (CHANGELOG): an older bd label / bd set-state against a new server cannot send allow_template and now gets 409 template_read_only on a template. bd comment, note, priority and tag keep their CLI-only pre-read guard. Co-Authored-By: Claude Opus 5.5 * fix(issueops): scope template read-only docs to updates, test label stand-down Review fixups for PR 7427 (adopt-pr review loop, iteration 1): - Contract: the role guard covers updates, not every mutation. The CHANGELOG headline, ErrTemplateReadOnly and CodeTemplateReadOnly now say so, and "Not yet moved" names claim, reopen, :casMetadata and the served addComment, which have no template check yet (a PATCH with claim: true refuses a template that POST :claim still claims). Close is #7425's. Follow-up: bd-jkp9v3. - Skew: the CapIssuesUpdateAllowTemplate client fallback covers updateIssue only. A batch update item gets no client-side check, so through an older bd serve it still edits a template; the CHANGELOG and both capability docs say so. - Tests: a template row now goes through the CLI on both tiers (TestEmbeddedLabelEditReports, TestProxiedServerLabelC). bd label add, bd label remove --prefix and bd set-state edit it (AllowTemplate); bd update --title and bd assign exit 1 with the template sentence and leave it unchanged. Mutants: dropping AllowTemplate from label.go and state.go reds the three edits on both tiers; disabling AuthorizeTemplateUpdate reds update and assign on both tiers. - Stale comments: runTagProxiedServer, applyUpdateProxiedOne, proxiedUpdateTarget, UpdateFlags (five booleans, not four) and the ItemUpdate notes in applyLabelEdit and bd label propagate's proxied route (UpdateItem has no AllowTemplate). Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 31 +++++ backend/conformance/batch_apply_contract.go | 69 +++++++++ .../conformance/lifecycle_update_contract.go | 131 ++++++++++++++++++ backend/conformance/role_bundle_cases.go | 4 + cmd/bd/BUILD.bazel | 1 + cmd/bd/assign.go | 8 +- cmd/bd/label.go | 9 +- cmd/bd/label_embedded_test.go | 72 +++++++++- cmd/bd/label_proxied_integration_test.go | 52 +++++++ cmd/bd/label_proxied_server.go | 5 +- cmd/bd/mutate_proxied_server.go | 24 ++-- cmd/bd/show_unit_helpers.go | 7 + cmd/bd/state.go | 3 + cmd/bd/update.go | 27 +++- cmd/bd/update_proxied_server.go | 34 ++--- cmd/bd/update_template_refusal_test.go | 35 +++++ internal/httpapi/BUILD.bazel | 1 + internal/httpapi/apigen/types.gen.go | 7 +- internal/httpapi/batch_apply.go | 5 + internal/httpapi/problem.go | 22 +++ internal/httpapi/routes.go | 15 +- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 47 ++++++- internal/httpapi/update.go | 10 +- internal/httpapi/update_template_test.go | 107 ++++++++++++++ .../httpapi/wireshape/testdata/golden.json | 6 + internal/httpclient/lifecycle.go | 40 ++++++ .../httpclient/served_batch_apply_test.go | 14 ++ .../httpclient/served_claim_lifecycle_test.go | 13 ++ internal/httpclient/wire/handshake.go | 14 +- internal/httpclient/wire/problem.go | 7 + internal/httpclient/wire/vocabulary_test.go | 4 +- internal/httpclient/wire/writes.go | 19 ++- internal/httpclient/write_ledger_test.go | 2 + internal/httpclient/write_roles_test.go | 100 ++++++++++++- .../storage/dolt/batch_apply_contract_test.go | 10 ++ .../dolt/lifecycle_update_contract_test.go | 6 + .../embeddeddolt/batch_apply_contract_test.go | 6 + .../lifecycle_update_contract_test.go | 6 + internal/storage/issueops/aggregate.go | 23 +++ internal/storage/issueops/batch_apply.go | 6 +- internal/storage/issueops/execution.go | 14 ++ .../issueops/version_completeness_test.go | 4 +- internal/storage/uow/batch_applier.go | 16 ++- .../storage/uow/batch_apply_contract_test.go | 6 + internal/storage/uow/issue_operations.go | 3 + .../issue_operations_claim_contract_test.go | 4 +- .../uow/lifecycle_update_contract_test.go | 6 + internal/validation/BUILD.bazel | 1 + internal/validation/issue.go | 3 +- issueops/BUILD.bazel | 1 + issueops/batchapplier.go | 4 +- issueops/issueops.go | 15 +- issueops/template_errors.go | 30 ++++ 54 files changed, 1038 insertions(+), 73 deletions(-) create mode 100644 cmd/bd/update_template_refusal_test.go create mode 100644 internal/httpapi/update_template_test.go create mode 100644 issueops/template_errors.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3497db5d1..77b8fc52d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -204,6 +204,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ones cannot fork. ### Fixed +- **An update of a template is now refused for every caller, not only + `bd update`.** The template guard used to be a pre-read in `cmd/bd`, so an + update through `bd serve` (`PATCH /v0/beads/issues/{id}`), an `update` item + of `issues:batchApply`, or a library caller of `issueops.Lifecycle.Update` or + `BatchApplier` edited a template that `bd update` refused. It now lives in + the role, in the update's own transaction, on every backend: any update + naming a template refuses with `*issueops.TemplateReadOnlyError` (matching + `ErrTemplateReadOnly`), whatever its patch and force flags, after the + compare-and-set guards. Over HTTP it is `409` `template_read_only`, and the + client rebuilds the same typed error. HTTP and library callers that edited + templates are now refused (pour the template instead) unless they set the + new `UpdateRequest.AllowTemplate` (`allow_template` on the PATCH body), which + stands the guard down for that one request. `bd update`, `bd assign` and the + proxied `bd tag` print the same line as before; `bd label` and + `bd set-state` set `AllowTemplate` and keep editing templates as they always + have. A batch create that makes a template and splices its metadata still + lands: the splice finishes the create. Version skew: a new `bd` against an + older `bd serve` (no `issues.update.allowTemplate` token in the handshake) + refuses a single template update itself before dialing, and never sends + `allow_template` to it. A batch `update` item gets no such check (its target + resolves on the server), so through an older `bd serve` it still edits a + template. An older `bd label` or `bd set-state` against a new `bd serve` + cannot send `allow_template`, so it now gets `409` `template_read_only` on a + template; upgrade the client. Not yet moved: `bd comment`, `bd note`, + `bd priority` and `bd tag` still refuse templates with their own pre-read in + `cmd/bd` (their direct-route writes do not go through `Lifecycle.Update`), so + for those four the guard lives only in the CLI. Claim, reopen, `:casMetadata` + and the served `addComment` have no template check at all yet (bd-jkp9v3): + `POST /v0/beads/issues/{id}:claim` (`issueops.Claimer`) still claims a + template that a `PATCH` with `claim: true` refuses. Close's guards move to + the role in their own change (#7425). - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index 49ed97a4f..6da619b21 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -2492,3 +2492,72 @@ func batchApplyHistoryMatching(t *testing.T, ctx context.Context, fixture BatchA } return count } + +// RunBatchApplyUpdateItemsRefuseATemplate pins that an update item answers to +// Lifecycle.Update's template guard: an item targeting a template refuses with +// an *ItemError carrying *TemplateReadOnlyError, whatever force flags it +// carries, and the request is all or nothing, so the create ahead of it does +// not land and the template is untouched. +func RunBatchApplyUpdateItemsRefuseATemplate(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + template := fixture.IssuePrefix + "-tmplupd-template" + bystander := fixture.IssuePrefix + "-tmplupd-bystander" + seeded := batchApplyIssue(template, "a template") + seeded.IsTemplate = true + if err := fixture.CreateIssue(ctx, seeded, "seed"); err != nil { + t.Fatalf("seed %s: %v", template, err) + } + + update := batchApplyUpdate(publicops.Ref{ID: template}, publicops.IssuePatch{ + Title: publicops.Field[string]{Set: true, Value: "edited"}, + Status: publicops.Field[publicops.Status]{Set: true, Value: types.StatusClosed}, + }) + update.Update.ForceClosePolicy = true + _, err := fixture.BatchApplier.ApplyBatch(ctx, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + ForceIDPrefix: true, + Items: []publicops.ApplyItem{ + batchApplyCreate("bystander", batchApplyIssue(bystander, "must not land beside a refusal")), + update, + }, + }) + var itemErr *publicops.ItemError + if !errors.As(err, &itemErr) { + t.Fatalf("an update item on template %s: error = %v, want an *ItemError", template, err) + } + if itemErr.Kind != publicops.ItemUpdate || itemErr.Index != 1 { + t.Errorf("ItemError = %#v, want the update item at index 1", itemErr) + } + var refusal *publicops.TemplateReadOnlyError + if !errors.As(err, &refusal) || refusal.IssueID != template { + t.Errorf("update item on template = %v, want *TemplateReadOnlyError naming %s", err, template) + } + assertBatchApplyRowCount(t, ctx, fixture, "issues", bystander, 0) + if got := batchApplyColumn(t, ctx, fixture, "title", template); got != "a template" { + t.Errorf("%s title after the refusal = %q, want it untouched", template, got) + } + if got := batchApplyColumn(t, ctx, fixture, "status", template); got != string(types.StatusOpen) { + t.Errorf("%s status after the refusal = %q, want it untouched", template, got) + } +} + +// RunBatchApplySplicesTheMetadataOfATemplateItCreates is the template guard's +// boundary: the one write a template legitimately takes inside a batch is a +// CREATE item that makes it and asks for a metadata splice. The splice runs as +// an update of the row the request itself is creating, after every id exists, +// and it must land — the template is being written, not modified — so a guard +// placed where the splice reaches it would refuse a template's own creation. +func RunBatchApplySplicesTheMetadataOfATemplateItCreates(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + created := fixture.IssuePrefix + "-tmplsplice-created" + fresh := batchApplyIssue(created, "a template this request creates") + fresh.IsTemplate = true + item := batchApplyCreate("created", fresh) + item.Create.MetadataRefs = map[string]publicops.Ref{"gc.self": {Key: "created"}} + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", ForceIDPrefix: true, Items: []publicops.ApplyItem{item}, + }) + if _, ok := batchApplyMetadataKey(t, ctx, fixture, created, "gc.self"); !ok { + t.Errorf("%s carries no gc.self key: the splice that finishes a template's own create must land", created) + } +} diff --git a/backend/conformance/lifecycle_update_contract.go b/backend/conformance/lifecycle_update_contract.go index 5219b494d..d58cc0536 100644 --- a/backend/conformance/lifecycle_update_contract.go +++ b/backend/conformance/lifecycle_update_contract.go @@ -2519,3 +2519,134 @@ func lifecycleUpdateHasEdge(dependencies []*types.Dependency, wantTarget string) } return false } + +// RunLifecycleUpdateRefusesATemplate pins Lifecycle.Update's template guard: +// templates are read-only, so EVERY update naming one refuses with +// *TemplateReadOnlyError — a field edit, a label edit, a claim, a status +// crossing — whatever force flags ride along, and nothing is written. It is the +// guard `bd update` used to apply as a CLI pre-read, which left bd serve and +// library callers editing templates bd refused. +// +// The refusal is asserted at the level a caller acts on — the typed error, its +// sentinel and the sentence bd prints, read off the typed error because a +// served refusal wraps it — and the row is read back to prove nothing moved. +// Two controls: the same patches land on an ordinary sibling, so the guard is +// not refusing the patch shapes; and a stale ExpectedVersion on the template +// still reports the mismatch, so the guard runs after the compare-and-set +// preconditions like every other update fence. +func RunLifecycleUpdateRefusesATemplate(t *testing.T, ctx context.Context, fixture LifecycleUpdateFixture) { + t.Helper() + template := fixture.IssuePrefix + "-lut-template" + sibling := fixture.IssuePrefix + "-lut-sibling" + for _, issue := range []*types.Issue{ + {ID: template, IsTemplate: true}, + {ID: sibling}, + } { + issue.Title, issue.Status, issue.Priority, issue.IssueType = issue.ID, types.StatusOpen, 2, types.TypeTask + if err := fixture.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + before, err := fixture.GetIssue(ctx, template) + if err != nil { + t.Fatalf("read seeded template %s: %v", template, err) + } + + requests := []struct { + name string + request publicops.UpdateRequest + }{ + {name: "field edit", request: publicops.UpdateRequest{ + Patch: publicops.IssuePatch{Title: publicops.Field[string]{Set: true, Value: "edited"}}, + }}, + {name: "label edit", request: publicops.UpdateRequest{ + Patch: publicops.IssuePatch{Labels: publicops.LabelPatch{Add: []string{"lut-tag"}}}, + }}, + {name: "claim", request: publicops.UpdateRequest{Claim: true}}, + {name: "forced status crossing", request: publicops.UpdateRequest{ + Patch: publicops.IssuePatch{Status: publicops.Field[publicops.Status]{Set: true, Value: types.StatusClosed}}, + ForceClosePolicy: true, + }}, + {name: "forced assignee transfer", request: publicops.UpdateRequest{ + Patch: publicops.IssuePatch{Assignee: publicops.Field[string]{Set: true, Value: "someone"}}, + ForceAssigneeTransfer: true, + }}, + } + for _, tc := range requests { + request := tc.request + request.Actor, request.IssueID = "writer", template + _, err := fixture.Lifecycle.Update(ctx, request) + var refusal *publicops.TemplateReadOnlyError + if !errors.As(err, &refusal) { + t.Fatalf("%s on template %s: err = %v, want *TemplateReadOnlyError", tc.name, template, err) + } + if refusal.IssueID != template || !errors.Is(err, publicops.ErrTemplateReadOnly) { + t.Errorf("%s refusal = %#v (%v), want IssueID %q matching ErrTemplateReadOnly", tc.name, refusal, err, template) + } + if want := "cannot modify template " + template + ": templates are read-only; use 'bd mol pour' to create a work item"; refusal.Error() != want { + t.Errorf("%s refusal reads %q, want %q", tc.name, refusal.Error(), want) + } + after, err := fixture.GetIssue(ctx, template) + if err != nil { + t.Fatalf("read template %s after the %s refusal: %v", template, tc.name, err) + } + if after.RowVersion != before.RowVersion || after.Title != before.Title || after.Status != before.Status || + after.Assignee != before.Assignee || len(after.Labels) != len(before.Labels) { + t.Errorf("template %s after the %s refusal = %+v, want it unchanged from %+v", template, tc.name, after, before) + } + + control := tc.request + control.Actor, control.IssueID = "writer", sibling + if _, err := fixture.Lifecycle.Update(ctx, control); err != nil { + t.Errorf("%s on the ordinary sibling %s: %v, want it to land", tc.name, sibling, err) + } + } + + stale := before.RowVersion + 1 + _, err = fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: "writer", IssueID: template, ExpectedVersion: &stale, + Patch: publicops.IssuePatch{Title: publicops.Field[string]{Set: true, Value: "edited"}}, + }) + if !errors.Is(err, publicops.ErrVersionMismatch) { + t.Errorf("stale-guarded update of template %s: err = %v, want ErrVersionMismatch ahead of the template guard", template, err) + } +} + +// RunLifecycleUpdateAllowTemplateEditsATemplate pins the template guard's one +// stand-down: an update with AllowTemplate — what bd label and bd set-state +// send, since they have always edited templates — lands on a template, and +// only that request is waived (the same patch without it still refuses). +func RunLifecycleUpdateAllowTemplateEditsATemplate(t *testing.T, ctx context.Context, fixture LifecycleUpdateFixture) { + t.Helper() + template := fixture.IssuePrefix + "-lua-template" + issue := &types.Issue{ID: template, Title: template, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, IsTemplate: true} + if err := fixture.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", template, err) + } + + result, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: "writer", IssueID: template, AllowTemplate: true, + Patch: publicops.IssuePatch{Labels: publicops.LabelPatch{Add: []string{"lua-tag"}}}, + }) + if err != nil { + t.Fatalf("AllowTemplate label edit on template %s: %v, want it to land", template, err) + } + if !result.Changed || result.Issue == nil || !lifecycleUpdateHasLabel(result.Issue.Labels, "lua-tag") { + t.Errorf("AllowTemplate label edit on template %s = %+v, want a change carrying lua-tag", template, result) + } + after, err := fixture.GetIssue(ctx, template) + if err != nil { + t.Fatalf("read template %s: %v", template, err) + } + if !after.IsTemplate { + t.Errorf("template %s is no longer a template after an AllowTemplate edit", template) + } + + _, err = fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: "writer", IssueID: template, + Patch: publicops.IssuePatch{Labels: publicops.LabelPatch{Add: []string{"lua-other"}}}, + }) + if !errors.Is(err, publicops.ErrTemplateReadOnly) { + t.Errorf("the same edit without AllowTemplate: err = %v, want ErrTemplateReadOnly", err) + } +} diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index b74a615a8..059a50056 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -58,6 +58,8 @@ var roleContractCases = []roleContract{ RunBatchApplyReplayMintsANewSetOfRows, RunBatchApplyDoesNotMutateTheCallerRequest, RunBatchApplyRefusesAnUnusableRequest, + RunBatchApplyUpdateItemsRefuseATemplate, + RunBatchApplySplicesTheMetadataOfATemplateItCreates, ), roleCases("BatchCloser", "BatchCloser()", oncePerRole, @@ -479,6 +481,8 @@ var roleContractCases = []roleContract{ RunLifecycleUpdateParentIDReplacesEveryParent, RunLifecycleUpdatePersistentPreservesUnversionedClass, RunLifecycleUpdateProvenanceLabelsHistory, + RunLifecycleUpdateRefusesATemplate, + RunLifecycleUpdateAllowTemplateEditsATemplate, ), roleCases("Memories", "Memories()", oncePerRole, diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 309ae4fc3..6dc8c19db 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1227,6 +1227,7 @@ go_test( "update_proxied_integration_test.go", "update_proxied_server_test.go", "update_stray_positional_test.go", + "update_template_refusal_test.go", "user_config_path_windows_test.go", "utils_unit_test.go", "vc_embedded_test.go", diff --git a/cmd/bd/assign.go b/cmd/bd/assign.go index 83b145f33..3ae73a54a 100644 --- a/cmd/bd/assign.go +++ b/cmd/bd/assign.go @@ -75,9 +75,8 @@ Examples: issueStore := result.Store - if err := validateIssueUpdatable(id, result.Issue); err != nil { - return HandleErrorRespectJSON("%s", err) - } + // The template guard is the role's (issueops.Lifecycle.Update); its + // refusal is printed below as this command always printed it. // bd-98s5c: bd assign is shorthand for an unguarded assignee update — // same live-claim fence as bd update -a. mc-zndi7.74: skipped when this @@ -118,6 +117,9 @@ Examples: return reported } } + if refusal, ok := templateReadOnlyRefusal(id, err); ok { + return HandleErrorRespectJSON("%s", refusal) + } return HandleErrorRespectJSON("updating %s: %v", id, err) } diff --git a/cmd/bd/label.go b/cmd/bd/label.go index 5f5a768db..fccbaaf4f 100644 --- a/cmd/bd/label.go +++ b/cmd/bd/label.go @@ -105,7 +105,9 @@ func resolveLabelTarget(ctx context.Context, id string) (string, error) { // commits them together, so the N calls collapse back to one transaction and // one history entry with no new role and no new request type. It is not in this // slice because it needs a cmd/bd accessor of its own and because its end gate -// runs a hierarchy and cycle walk a label-only request has no use for. +// runs a hierarchy and cycle walk a label-only request has no use for. It also +// needs a template stand-down first: UpdateItem has no AllowTemplate, so an +// ItemUpdate refuses the templates this function edits today (bd-jkp9v3). func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, operation string) error { lifecycle, err := openIssueLifecycle() if err != nil { @@ -168,6 +170,9 @@ func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, ope Actor: currentActor(), IssueID: issueID, Patch: patch, + // bd label has always edited templates; the role's template + // guard stands down for it (issueops.UpdateRequest.AllowTemplate). + AllowTemplate: true, }) if uerr != nil { gerund := labelOperationGerund(operation) @@ -364,6 +369,8 @@ func removeLabelsByPrefix(ctx context.Context, issueIDs []string, prefix string, Actor: currentActor(), IssueID: target.issueID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Remove: target.labels}}, + // As above: bd label edits templates by design. + AllowTemplate: true, }) if uerr != nil { return HandleErrorRespectJSON("label removing (prefix): %s: %v", target.issueID, uerr) diff --git a/cmd/bd/label_embedded_test.go b/cmd/bd/label_embedded_test.go index b0e158c63..e4f1e88a4 100644 --- a/cmd/bd/label_embedded_test.go +++ b/cmd/bd/label_embedded_test.go @@ -7,9 +7,13 @@ import ( "fmt" "os" "os/exec" + "path/filepath" + "slices" "strings" "sync" "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" ) // bdLabel runs "bd label" with the given args and returns stdout. @@ -99,6 +103,28 @@ func bdLabelListAllJSON(t *testing.T, bd, dir string) []map[string]interface{} { return results } +// markIssueTemplate makes an existing issue a template (is_template = 1) with +// raw SQL and commits it, so a test can put a template in front of the CLI +// without cooking a formula. +func markIssueTemplate(t *testing.T, beadsDir, database, issueID string) { + t.Helper() + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), filepath.Join(beadsDir, "embeddeddolt"), database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer cleanup() + res, err := db.ExecContext(t.Context(), "UPDATE issues SET is_template = 1 WHERE id = ?", issueID) + if err != nil { + t.Fatalf("mark %s a template: %v", issueID, err) + } + if n, err := res.RowsAffected(); err != nil || n != 1 { + t.Fatalf("mark %s a template: %d rows affected (err %v), want 1", issueID, n, err) + } + if _, err := db.ExecContext(t.Context(), "CALL DOLT_COMMIT('-Am', 'test: mark a template')"); err != nil { + t.Fatalf("commit the template mark: %v", err) + } +} + // TestEmbeddedLabelAddRemove was split from TestEmbeddedLabel (originally // ~360s, measured under --config=embedded) into 3 top-level tests over // disjoint subtest groups, for CI shard balance (see @@ -361,7 +387,7 @@ func TestEmbeddedLabelEditReports(t *testing.T) { t.Parallel() bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "tl") + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tl") // Two issues carrying DIFFERENT matching sets: the report is per issue and // each issue's set is resolved from its own labels, so a refactor that @@ -614,6 +640,50 @@ func TestEmbeddedLabelEditReports(t *testing.T) { } } }) + + // A template is read-only to an update — issueops.Lifecycle.Update refuses + // it on every route — but bd label and bd set-state edit templates by + // design: they set UpdateRequest.AllowTemplate. One template pins both + // sides end to end. The stand-down verbs are checked with t.Errorf rather + // than a fatal helper, so a lost AllowTemplate names every verb that lost + // it. + t.Run("template_label_edits_pass_update_and_assign_refuse", func(t *testing.T) { + tpl := bdCreate(t, bd, dir, "Template under edit", "--type", "task", + "--label", "pool:refused:reason-a") + markIssueTemplate(t, beadsDir, "tl", tpl.ID) + + for _, args := range [][]string{ + {"label", "add", tpl.ID, "tpl-added"}, + {"label", "remove", tpl.ID, "--prefix", "pool:refused:"}, + {"set-state", tpl.ID, "phase=planning"}, + } { + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Errorf("bd %s on a template: %v\n%s", strings.Join(args, " "), err, out) + } + } + labels := bdLabelListJSON(t, bd, dir, tpl.ID) + slices.Sort(labels) + if want := []string{"phase:planning", "tpl-added"}; !slices.Equal(labels, want) { + t.Errorf("template labels after label add, label remove --prefix and set-state = %v, want %v", labels, want) + } + + want := "cannot modify template " + tpl.ID + ": templates are read-only; use 'bd mol pour' to create a work item" + for _, args := range [][]string{ + {"update", tpl.ID, "--title", "Edited template"}, + {"assign", tpl.ID, "someone-else"}, + } { + if out, code := bdRunFailCode(t, bd, dir, args...); code != 1 || !strings.Contains(out, want) { + t.Errorf("bd %s on a template: exit %d, output:\n%s\nwant exit 1 and %q", strings.Join(args, " "), code, out, want) + } + } + if got := bdShow(t, bd, dir, tpl.ID); got.Title != "Template under edit" || got.Assignee != "" || !got.IsTemplate { + t.Errorf("the refused update and assign changed the template: title=%q assignee=%q is_template=%v", + got.Title, got.Assignee, got.IsTemplate) + } + }) } // TestEmbeddedLabelPropagateRename was split from TestEmbeddedLabel diff --git a/cmd/bd/label_proxied_integration_test.go b/cmd/bd/label_proxied_integration_test.go index 7cfd1d94d..2d820d97f 100644 --- a/cmd/bd/label_proxied_integration_test.go +++ b/cmd/bd/label_proxied_integration_test.go @@ -5,6 +5,9 @@ package main import ( "context" "encoding/json" + "errors" + "os/exec" + "slices" "sort" "strings" "testing" @@ -581,4 +584,53 @@ func TestProxiedServerLabelC(t *testing.T) { t.Errorf("labels (permanent) count = %d, want 0 — propagated wisp label must not leak into the permanent table", permCount) } }) + + // The proxied twin of TestEmbeddedLabelEditReports' + // template_label_edits_pass_update_and_assign_refuse, over the unit-of-work + // leg: bd label and bd set-state stand the template guard down + // (UpdateRequest.AllowTemplate), bd update and bd assign are refused with + // the CLI's sentence and exit 1. + t.Run("template_label_edits_pass_update_and_assign_refuse", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "lt") + tpl := bdProxiedCreate(t, bd, p.dir, "Template under edit", "--type", "task", + "--label", "pool:refused:reason-a") + res, err := openProxiedDB(t, p).Exec("UPDATE issues SET is_template = 1 WHERE id = ?", tpl.ID) + if err != nil { + t.Fatalf("mark %s a template: %v", tpl.ID, err) + } + if n, err := res.RowsAffected(); err != nil || n != 1 { + t.Fatalf("mark %s a template: %d rows affected (err %v), want 1", tpl.ID, n, err) + } + + for _, args := range [][]string{ + {"label", "add", tpl.ID, "tpl-added"}, + {"label", "remove", tpl.ID, "--prefix", "pool:refused:"}, + {"set-state", tpl.ID, "phase=planning"}, + } { + if stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, args...); err != nil { + t.Errorf("bd %s on a template: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), err, stdout, stderr) + } + } + if got, want := bdProxiedLabelListJSON(t, bd, p.dir, tpl.ID), []string{"phase:planning", "tpl-added"}; !slices.Equal(got, want) { + t.Errorf("template labels after label add, label remove --prefix and set-state = %v, want %v", got, want) + } + + want := "cannot modify template " + tpl.ID + ": templates are read-only; use 'bd mol pour' to create a work item" + for _, args := range [][]string{ + {"update", tpl.ID, "--title", "Edited template"}, + {"assign", tpl.ID, "someone-else"}, + } { + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, args...) + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 || !strings.Contains(stdout+stderr, want) { + t.Errorf("bd %s on a template: err %v\nstdout:\n%s\nstderr:\n%s\nwant exit 1 and %q", + strings.Join(args, " "), err, stdout, stderr, want) + } + } + if got := bdProxiedShow(t, bd, p.dir, tpl.ID); got.Title != "Template under edit" || got.Assignee != "" || !got.IsTemplate { + t.Errorf("the refused update and assign changed the template: title=%q assignee=%q is_template=%v", + got.Title, got.Assignee, got.IsTemplate) + } + }) } diff --git a/cmd/bd/label_proxied_server.go b/cmd/bd/label_proxied_server.go index 0bf9215f0..72b67c467 100644 --- a/cmd/bd/label_proxied_server.go +++ b/cmd/bd/label_proxied_server.go @@ -205,7 +205,10 @@ func runLabelPropagateProxiedServer(ctx context.Context, args []string) error { // keeps both is issueops.BatchApplier.ApplyBatch with one ItemUpdate per // child carrying this same label patch, and it is blocked on a cmd/bd // accessor for that role. That is the follow-up this waiver names - // (ga-2ltro.12); it is the last one on this list that WRITES. + // (ga-2ltro.12); it is the last one on this list that WRITES. It also + // needs a template stand-down first: UpdateItem has no AllowTemplate, so + // an ItemUpdate refuses the template children this loop labels today + // (bd-jkp9v3). for _, child := range children { var e error if child.Ephemeral { diff --git a/cmd/bd/mutate_proxied_server.go b/cmd/bd/mutate_proxied_server.go index fa72c238e..9470c01fa 100644 --- a/cmd/bd/mutate_proxied_server.go +++ b/cmd/bd/mutate_proxied_server.go @@ -87,9 +87,8 @@ func proxiedAssign(ctx context.Context, id, assignee string, force bool, ifRevis if rerr != nil { return struct{}{}, fmt.Errorf("resolving %s: %w", id, rerr) } - if verr := validateIssueUpdatable(id, current); verr != nil { - return struct{}{}, verr - } + // The template guard is the role's, enforced by the Lifecycle.Update + // below; runAssignProxiedServer prints its refusal as before. // mc-zndi7.74: skipped when this pre-read is already stale against an // active --if-revision guard, so a lost race reports precondition_failed // from the guarded write below instead of this policy refusal — see @@ -137,6 +136,9 @@ func runAssignProxiedServer(ctx context.Context, args []string, force bool, ifRe return reported } } + if refusal, ok := templateReadOnlyRefusal(id, err); ok { + err = refusal + } return HandleErrorRespectJSON("assign %s: %v", id, err) } if jsonOutput { @@ -211,10 +213,10 @@ func runNoteProxiedServer(ctx context.Context, id, noteText string) error { // UpdateRequest.IssuePlaneOnly stays false, so the role resolves the plane // inside its own transaction and there is no boolean here to get backwards. // -// The two reads it still makes are front-door work rather than plumbing. -// issueops.Reader.Get supplies the issue the template guard needs — the roles -// have no opinion about templates, and the direct route refuses one — and the -// role takes an exact id by contract, which Get's issue-then-wisp lookup is. +// The read it still makes is front-door work rather than plumbing: the role +// takes an exact id by contract, and issueops.Reader.Get's issue-then-wisp +// lookup supplies one. The template guard is the role's: Lifecycle.Update +// refuses a template inside its own transaction. // The label arrives already normalized: tag.go does that before choosing a // route, so this path and the direct one cannot disagree about what was stored. func runTagProxiedServer(ctx context.Context, id, label string) error { @@ -229,9 +231,8 @@ func runTagProxiedServer(ctx context.Context, id, label string) error { if err != nil { return HandleErrorRespectJSON("tag %s: resolving %s: %v", id, id, err) } - if verr := validateIssueUpdatable(id, &details.Issue); verr != nil { - return HandleErrorRespectJSON("tag %s: %v", id, verr) - } + // The template guard is the role's, enforced by the Lifecycle.Update + // below, whose refusal prints as this route always printed it. lifecycle, err := openIssueLifecycle() if err != nil { @@ -250,6 +251,9 @@ func runTagProxiedServer(ctx context.Context, id, label string) error { Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Add: []string{label}}}, }) if err != nil { + if refusal, ok := templateReadOnlyRefusal(id, err); ok { + err = refusal + } return HandleErrorRespectJSON("tag %s: %v", id, err) } commandDidWrite.Store(true) diff --git a/cmd/bd/show_unit_helpers.go b/cmd/bd/show_unit_helpers.go index 4e2fcd6e2..74c0e9705 100644 --- a/cmd/bd/show_unit_helpers.go +++ b/cmd/bd/show_unit_helpers.go @@ -11,6 +11,13 @@ import ( ) // validateIssueUpdatable checks if an issue can be updated. +// +// It is the CLI-only template guard that bd comment, bd note, bd priority and +// bd tag (and the proxied comment and mutate routes) still run as a pre-read: +// their writes do not go through issueops.Lifecycle.Update, whose role rule +// (storage issueops.AuthorizeTemplateUpdate) guards bd update and bd assign on +// every route. Until those commands move onto the role, this is their only +// guard. // Uses the centralized validation package for consistency. func validateIssueUpdatable(id string, issue *types.Issue) error { // Note: We use NotTemplate() directly instead of ForUpdate() to maintain diff --git a/cmd/bd/state.go b/cmd/bd/state.go index b689b0f28..3d0b56a78 100644 --- a/cmd/bd/state.go +++ b/cmd/bd/state.go @@ -260,6 +260,9 @@ func runSetState(ctx context.Context, issueID, dimension, newValue, reason strin Add: []string{newLabel}, Remove: removeStateLabel(oldLabel), }}, + // bd set-state has always edited templates; the role's template guard + // stands down for it (issueops.UpdateRequest.AllowTemplate). + AllowTemplate: true, }); err != nil { return HandleErrorRespectJSON("setting %s: %v", dimension, err) } diff --git a/cmd/bd/update.go b/cmd/bd/update.go index 09e8edb64..95e013d9f 100644 --- a/cmd/bd/update.go +++ b/cmd/bd/update.go @@ -493,12 +493,9 @@ pointless).`, issue := result.Issue issueStore := result.Store - if err := validateIssueUpdatable(id, issue); err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - recordFailure(id, err.Error()) - closeIfUnmutated(result) - continue - } + // The template guard is the role's (issueops.Lifecycle.Update), + // enforced inside the mutation on every route; its refusal is + // printed below as the line this command always printed. // bd-98s5c: an unguarded assignee update must not silently // overwrite another actor's live claim. Skipped under @@ -577,7 +574,10 @@ pointless).`, } } failureText := fmt.Sprintf("updating issue: %v", updateErr) - if errors.Is(updateErr, issueops.ErrNotesOverwrite) { + if refusal, ok := templateReadOnlyRefusal(id, updateErr); ok { + failureText = refusal.Error() + fmt.Fprintf(os.Stderr, "%s\n", refusal) + } else if errors.Is(updateErr, issueops.ErrNotesOverwrite) { // The contract's AuthorizeNotesOverwrite fence refused // inside the mutation transaction. Print the advice, not // the raw sentinel. @@ -1117,3 +1117,16 @@ func init() { updateCmd.ValidArgsFunction = issueIDCompletion rootCmd.AddCommand(updateCmd) } + +// templateReadOnlyRefusal reports whether err is the update role's template +// refusal (issueops.Lifecycle.Update, any route) and, when it is, returns it +// spelled against id — the argument as the caller typed it, which is what this +// CLI's template sentence has always named. It is found with errors.Is because +// the refusal may arrive wrapped by the unit of work or inside the served +// problem envelope. +func templateReadOnlyRefusal(id string, err error) (error, bool) { + if !errors.Is(err, issueops.ErrTemplateReadOnly) { + return nil, false + } + return &issueops.TemplateReadOnlyError{IssueID: id}, true +} diff --git a/cmd/bd/update_proxied_server.go b/cmd/bd/update_proxied_server.go index 5d74583b2..69ba47fb2 100644 --- a/cmd/bd/update_proxied_server.go +++ b/cmd/bd/update_proxied_server.go @@ -87,11 +87,12 @@ func proxiedIssueLifecycle() (issueops.Lifecycle, error) { // applyUpdateProxiedOne applies one issue's update — plain or --claim — // through issueops.Lifecycle. What stays here is this surface's own protocol: -// the template guard, the advisory reassign pre-read, the per-id failure -// taxonomy the multi-id batch needs and the notes-overwrite warning. Hooks are -// NOT among them: they fire from the write plumbing now (the notifying provider -// wired in main.go), which is what makes an update fire the same events here as -// it does on the embedded path. +// the advisory reassign pre-read, the per-id failure taxonomy the multi-id +// batch needs (a template refusal is one of its verdicts; the guard itself is +// the role's) and the notes-overwrite warning. Hooks are NOT among them: they +// fire from the write plumbing now (the notifying provider wired in main.go), +// which is what makes an update fire the same events here as it does on the +// embedded path. // // Provenance carries the commit message this path has always written, so `bd // dolt log` reads the same after the move as before it. The plane is @@ -189,13 +190,12 @@ func applyUpdateProxiedOne(ctx context.Context, id string, in *updateInput) (*ty } // proxiedUpdateTarget reads the row an update is about through the query role, -// for the things the mutation's own result cannot answer: whether the target is -// a template, whether an unguarded assignee edit is about to take the issue -// from a live foreign holder, whether --notes is about to replace existing -// notes, and whether --defer="" should also clear a deferred status. It no -// longer reads the pre-state to decide a hook: a status-crossing update fires -// on_update and nothing else, from the plumbing, exactly as it does on the -// embedded path. +// for the things the mutation's own result cannot answer: whether an unguarded +// assignee edit is about to take the issue from a live foreign holder, whether +// --notes is about to replace existing notes, and whether --defer="" should +// also clear a deferred status. It no longer reads the pre-state to decide a +// hook: a status-crossing update fires on_update and nothing else, from the +// plumbing, exactly as it does on the embedded path. func proxiedUpdateTarget(ctx context.Context, id string, in *updateInput) (*types.Issue, *updateIDFailure) { rd, err := proxiedIssueReader() if err != nil { @@ -212,10 +212,8 @@ func proxiedUpdateTarget(ctx context.Context, id string, in *updateInput) (*type return nil, &updateIDFailure{ID: id, Error: fmt.Sprintf("resolving issue: %v", err)} } current := &details.Issue - if err := validateIssueUpdatable(id, current); err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - return nil, &updateIDFailure{ID: id, Error: err.Error()} - } + // The template guard is the role's, enforced inside the mutation; + // proxiedUpdateFailure prints its refusal as this route always did. // bd-98s5c: an unguarded assignee update must not silently overwrite // another actor's live claim. Skipped under --if-assignee, whose CAS names // the holder explicitly (park stays possible without --force), and under @@ -266,6 +264,10 @@ func proxiedClaimPoolAliases(ctx context.Context) func() []string { // longer owns, so both now read as the generic update failure. The id still // fails, loudly and non-zero. func proxiedUpdateFailure(id string, claim bool, err error) *updateIDFailure { + if refusal, ok := templateReadOnlyRefusal(id, err); ok { + fmt.Fprintf(os.Stderr, "%s\n", refusal) + return &updateIDFailure{ID: id, Error: refusal.Error()} + } switch { case errors.Is(err, storage.ErrNotFound): fmt.Fprintf(os.Stderr, "Issue %s not found\n", id) diff --git a/cmd/bd/update_template_refusal_test.go b/cmd/bd/update_template_refusal_test.go new file mode 100644 index 000000000..630e39822 --- /dev/null +++ b/cmd/bd/update_template_refusal_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "errors" + "fmt" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// TestTemplateReadOnlyRefusalKeepsTheCLISentence pins what `bd update`, +// `bd assign` and the proxied `bd tag` print now that the template guard is the +// role's: the sentence they printed when it was a CLI pre-read, naming the id +// as the caller typed it — however the refusal travelled (bare from the store, +// wrapped by the unit of work, or rebuilt by the served client against the +// resolved id). +func TestTemplateReadOnlyRefusalKeepsTheCLISentence(t *testing.T) { + const typed = "tp-ab" + want := "cannot modify template " + typed + ": templates are read-only; use 'bd mol pour' to create a work item" + for name, err := range map[string]error{ + "bare": &issueops.TemplateReadOnlyError{IssueID: "tp-abc123"}, + "wrapped": fmt.Errorf("update tp-abc123: %w", &issueops.TemplateReadOnlyError{IssueID: "tp-abc123"}), + } { + refusal, ok := templateReadOnlyRefusal(typed, err) + if !ok { + t.Fatalf("%s: templateReadOnlyRefusal did not recognize %v", name, err) + } + if refusal.Error() != want { + t.Errorf("%s: refusal reads %q, want %q", name, refusal.Error(), want) + } + } + if _, ok := templateReadOnlyRefusal(typed, errors.New("something else")); ok { + t.Errorf("an unrelated error was taken for the template refusal") + } +} diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 27efb854f..fab98a6f5 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -106,6 +106,7 @@ go_test( "sweep_test.go", "timed_provider_binding_test.go", "tree_test.go", + "update_template_test.go", "update_test.go", "wire_bijection_test.go", "wire_revision_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 4fea37994..b09477c7c 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1047,7 +1047,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -1607,7 +1607,7 @@ type Problem struct { // IT IS OPTIONAL. A refusal that could not name its blockers omits it and keeps the generic `detail`; absence means "this refusal did not name them", never "nothing blocks it". Re-read the issue's dependencies then. Blockers *[]Blocker `json:"blockers,omitempty"` - // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). + // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `template_read_only` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). Code string `json:"code"` // DeclaredLater With `invalid_argument` on a batch operation whose items may name each other: whether the unresolvable key IS declared by the request, at a LATER index. @@ -2096,6 +2096,9 @@ type UpdateIssueRequest struct { // Actor Who is editing the issue. `ClaimRequest.actor`'s rules exactly: the server trims it, then refuses an empty result, anything longer than 256 BYTES (the `maxLength` above counts characters — the byte limit is the binding one), and any control character including newline. The value reaches the history entry's attribution and the storage commit message, so an unvalidated newline would forge audit-trail lines. Actor string `json:"actor"` + // AllowTemplate The caller edits a template deliberately, so the template read-only refusal (`409 template_read_only`) stands down for this request. `bd label` and `bd set-state` send it: they have always written to templates. It bypasses nothing else, and has no effect on an issue that is not a template. Advertised by the `issues.update.allowTemplate` capability token; a server that predates the member refuses it as an unknown parameter, and such a server never applies the template guard either. + AllowTemplate *bool `json:"allow_template,omitempty"` + // Claim Claims the issue for `actor` in the same transaction as `patch`: `bd update --claim`, served by the same role. The claim sets `assignee` to `actor` and `status` to `in_progress`, then the patch applies, so a `patch.assignee` or `patch.status` overrides the claim's value. Eligibility is `{id}:claim`'s: a claimable status, and unassigned, already held by `actor`, or held by a configured claim pool. Held by `actor` and `in_progress` already is an idempotent success. A refusal is `409 already_claimed` or `409 not_claimable` naming `claim`, and writes nothing — the patch included. With `claim: true` the `patch` may be empty. It must not be combined with `expected_assignee`, `expected_status` or `force_assignee_transfer`; a request that does is a `400` naming `claim`. Claim *bool `json:"claim,omitempty"` diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index 723ce8615..8b2e3fcba 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -1054,6 +1054,11 @@ func (s *Server) failApplyBatch(w http.ResponseWriter, r *http.Request, request s.fail(w, r, at(closeBlockedResult(err, "an item closes a blocked issue", "clear the blocker, or send the item's force flag"), "")) + // An update item naming a template: the shared mapping plus the item + // members. + case errors.Is(err, issueops.ErrTemplateReadOnly): + s.fail(w, r, at(ClassifyError(err), "")) + case errors.Is(err, storage.ErrAlreadyClaimed): res := at(newResult(CodeAlreadyClaimed, "an update transfers work away from a live foreign owner; send `force_assignee_transfer`, or guard with `expected_assignee`"), "assignee") diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index 87618d3bd..a7694e09a 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -136,6 +136,16 @@ const ( // notes and the patched ones), not about a foreign actor's identity a // client might need to display. CodeNotesOverwrite Code = "notes_overwrite_refused" + // CodeTemplateReadOnly is a guarded mutation that names a template; + // operationCodes lists the operations that answer it, and not every verb + // guards templates yet (bd-jkp9v3). Templates are read-only — work comes + // out of one by pouring it, which creates new issues — so no force flag + // waives it: the same request refuses for as long as the row is a + // template. The one stand-down is updateIssue's `allow_template`, a + // deliberate template edit (bd label, bd set-state). + // A 409 for CodeNotClosable's reason: the body is well-formed and the + // STATE of the named row refuses it. + CodeTemplateReadOnly Code = "template_read_only" // CodeDependencyCycle covers BOTH never-makes-progress refusals a requested // edge set can earn: a scheduling cycle, and a blocking edge against the // issue's own ancestor or descendant. They are one code because they have @@ -262,6 +272,7 @@ var codeStatus = map[Code]int{ CodeNotClaimable: http.StatusConflict, CodeNotClosable: http.StatusConflict, CodeNotesOverwrite: http.StatusConflict, + CodeTemplateReadOnly: http.StatusConflict, CodeNotReleasable: http.StatusConflict, CodeDependencyCycle: http.StatusConflict, CodeDependencyExists: http.StatusConflict, @@ -897,9 +908,14 @@ var operationCodes = map[string][]Code{ // // not_claimable arrived with `claim`, beside the already_claimed it shares // with the fence: a refused claim answers as claimIssue does. + // + // template_read_only is the role refusing an update of a template: + // templates are read-only, no force flag waives it, and only + // `allow_template` stands it down. OpUpdateIssue: { CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, CodePreconditionFailed, CodeNotClosable, CodeAlreadyClaimed, CodeNotesOverwrite, + CodeTemplateReadOnly, CodeNotClaimable, CodeDependencyCycle, CodeDependencyExists, CodeBusy, CodeDBUnavailable, CodeInternal, }, @@ -962,6 +978,7 @@ var operationCodes = map[string][]Code{ OpApplyBatch: { CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, CodePreconditionFailed, CodeNotClosable, CodeAlreadyClaimed, CodeNotesOverwrite, CodeAlreadyExists, + CodeTemplateReadOnly, CodeDependencyCycle, CodeDependencyExists, CodeBusy, CodeDBUnavailable, CodeInternal, }, @@ -1543,6 +1560,11 @@ func ClassifyError(err error) Result { case errors.Is(err, issueops.ErrCloseBlocked): return closeBlockedResult(err, "issue is blocked", "clear the blocker or close with force") + // The detail is this server's own words rather than the role's message, + // for the reason Problem.detail gives. + case errors.Is(err, issueops.ErrTemplateReadOnly): + return newResult(CodeTemplateReadOnly, "this issue is a template, and templates are read-only; pour it to create work") + case errors.Is(err, ErrBusy): res := newResult(CodeBusy, "") res.RetryAfterSeconds = retryAfterSaturation diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 1dc78fa94..f0c7b69d9 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -161,6 +161,19 @@ const CapIssuesSweepLimit = "issues.sweep.limit" // predate it; it refuses locally before the dial when the token is absent. const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" +// CapIssuesUpdateAllowTemplate is the behavior capability that advertises +// two things together: updateIssue enforces the role's template read-only +// refusal, and UpdateIssueRequest's `allow_template` member (update.go) stands +// it down for one request (the caller edits a template deliberately). An +// older server predating this token answers the member with +// `400 invalid_argument`/`unknown_parameter` and applies no template guard, +// so a client that does not see the token refuses a template update itself +// before the dial and sends the request without the member. That fallback +// covers updateIssue only: an applyBatch `update` item gets no client-side +// check (its target resolves on the server), so through an older server it +// still edits a template (bd-jkp9v3). +const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -922,7 +935,7 @@ func (r route) specPathOf() string { var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, - CapBatchApplyDepAddLineage, + CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 6f787874c..0460d6231 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -637,7 +637,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // oracle rather than a second call to the code under test. want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", - "issues.batchApply.depAddLineage") + "issues.batchApply.depAddLineage", "issues.update.allowTemplate") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 0c087a175..6d9c5b354 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -2501,10 +2501,11 @@ paths: '409': description: >- The request is well-formed and the STATE refuses it, and NOTHING WAS - WRITTEN. Six codes, each inherited from the operation that already - publishes it rather than minted here, and described in the - paragraphs below — `dependency_cycle` and `dependency_exists` share - one, because they are one member reaching the graph. + WRITTEN. Eight codes, described in the paragraphs below; all but + `template_read_only` are inherited from the operation that already + publishes them rather than minted here, and `dependency_cycle` and + `dependency_exists` share one paragraph, because they are one member + reaching the graph. `already_claimed` and `not_claimable` with `param: "claim"` are a @@ -2557,9 +2558,18 @@ paths: both of which write blocking edges and do carry it: this operation publishes no member that writes one. What you get here is the plain scheduling cycle, every time. + + + `template_read_only` is an update of a template. Templates are + read-only — work comes out of one by pouring it — so every update + naming one refuses, whatever the patch and force flags, unless it + sets `allow_template` (a deliberate template edit). + It is checked after the guards above, so a stale guard still answers + `precondition_failed`. x-bd-codes: [already_claimed, dependency_cycle, dependency_exists, - not_claimable, not_closable, notes_overwrite_refused, precondition_failed] + not_claimable, not_closable, notes_overwrite_refused, precondition_failed, + template_read_only] content: application/problem+json: schema: @@ -4328,6 +4338,11 @@ paths: per-item bypasses, and a forced item never earns this. + `template_read_only` is an `update` item whose target is a + template, exactly as `PATCH /v0/beads/issues/{id}` answers it; no + item member bypasses it. + + `already_claimed` is the assignee fence: an `update` whose `patch.assignee` transfers work away from a live foreign in-progress owner. `update.force_assignee_transfer` is the bypass, and @@ -4357,7 +4372,8 @@ paths: a malformed request. x-bd-codes: [already_claimed, already_exists, dependency_cycle, - dependency_exists, not_closable, notes_overwrite_refused, precondition_failed] + dependency_exists, not_closable, notes_overwrite_refused, precondition_failed, + template_read_only] content: application/problem+json: schema: @@ -8065,7 +8081,11 @@ components: `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than - a `400` naming the member unknown. The + a `400` naming the member unknown, and + `issues.update.allowTemplate`, which announces that + `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see + `UpdateIssueRequest`) rather than a `400` naming the member + unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. @@ -11597,6 +11617,18 @@ components: `expected_assignee` exemption: there is no compare-and-set that authorizes a notes overwrite, so combining the two is legal and each answers its own question. + allow_template: + type: boolean + default: false + description: >- + The caller edits a template deliberately, so the template read-only + refusal (`409 template_read_only`) stands down for this request. + `bd label` and `bd set-state` send it: they have always written to + templates. It bypasses nothing else, and has no effect on an issue + that is not a template. Advertised by the + `issues.update.allowTemplate` capability token; a server that + predates the member refuses it as an unknown parameter, and such a + server never applies the template guard either. IssuePatchBody: type: object @@ -12153,6 +12185,7 @@ components: configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), + `template_read_only` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), diff --git a/internal/httpapi/update.go b/internal/httpapi/update.go index 35feb3932..64df715ce 100644 --- a/internal/httpapi/update.go +++ b/internal/httpapi/update.go @@ -39,7 +39,8 @@ const ( var ( updateRequestMembers = []string{ "actor", updateClaimMember, "expected_assignee", "expected_status", "expected_version", - "force_assignee_transfer", "force_close_policy", "force_notes_overwrite", updatePatchMember, + "force_assignee_transfer", "force_close_policy", "force_notes_overwrite", "allow_template", + updatePatchMember, } issuePatchMembers = []string{ "title", "description", "design", "acceptance_criteria", @@ -181,6 +182,12 @@ func (s *Server) updateRequest(w http.ResponseWriter, r *http.Request, id string if !ok { return issueops.UpdateRequest{}, false } + // allow_template is passed through: the role owns the template guard and + // what standing it down means. + allowTemplate, ok := s.booleanMember(w, r, members, "allow_template") + if !ok { + return issueops.UpdateRequest{}, false + } if forceNotesOverwrite && !patch.Notes.Set { s.fail(w, r, InvalidArgument("force_notes_overwrite", ReasonInvalidValue, "`force_notes_overwrite` bypasses the fence on a NOTES REPLACEMENT; send `patch.notes` with it")) @@ -244,6 +251,7 @@ func (s *Server) updateRequest(w http.ResponseWriter, r *http.Request, id string ForceClosePolicy: forceClosePolicy, ForceAssigneeTransfer: forceAssigneeTransfer, ForceNotesOverwrite: forceNotesOverwrite, + AllowTemplate: allowTemplate, Provenance: updateProvenance, }, true } diff --git a/internal/httpapi/update_template_test.go b/internal/httpapi/update_template_test.go new file mode 100644 index 000000000..505535c2a --- /dev/null +++ b/internal/httpapi/update_template_test.go @@ -0,0 +1,107 @@ +package httpapi + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// The template guard is the ROLE's (issueops.Lifecycle.Update, and the batch +// applier's update items, which share its body), so these cases drive a fake +// role that refuses the way the real one does and pin what the wire makes of +// it. The real role behind a real server is the served leg of the conformance +// contracts (internal/httpclient: TestServedLifecycleUpdateRefusesATemplate and +// TestServedBatchApplyUpdateItemsRefuseATemplate), which also proves the client +// rebuilds the typed error. + +// templateRefusal is the refusal as a unit-of-work backend returns it: wrapped, +// so the mapping has to find it by type rather than by being handed the value. +func templateRefusal() error { + return fmt.Errorf("update bd-1: %w", &issueops.TemplateReadOnlyError{IssueID: "bd-1"}) +} + +func assertTemplateDetailIsOurs(t *testing.T, detail any) { + t.Helper() + text, _ := detail.(string) + if text == "" || strings.Contains(text, "cannot modify template") { + t.Errorf("detail = %q, want this server's own sentence rather than the role's", text) + } +} + +func TestUpdateOfATemplateIsATypedConflict(t *testing.T) { + lifecycle := &roleLifecycle{updateErr: templateRefusal()} + ts := newUpdateServer(t, lifecycle) + + resp := ts.updateIssue(t, updatePath, `{"actor":"alice","patch":{"title":"edited"},"force_close_policy":true}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeTemplateReadOnly) { + t.Errorf("code = %v, want %s", body["code"], CodeTemplateReadOnly) + } + assertTemplateDetailIsOurs(t, body["detail"]) + // The handler decides nothing: the request reached the role whole, force + // flag included, and the refusal is the role's. + if got := lifecycle.updateRequests(); len(got) != 1 || !got[0].ForceClosePolicy { + t.Fatalf("the role received %+v, want one request carrying ForceClosePolicy", got) + } +} + +func TestApplyBatchUpdateItemOnATemplateIsATypedConflict(t *testing.T) { + applier := &roleBatchApplier{err: itemErr(1, issueops.ItemUpdate, "", "bd-1", templateRefusal())} + ts := newApplyBatchServer(t, applier) + + resp := ts.claim(t, batchApplyPath, `{"actor":"alice","items":[{"kind":"create","create":{"title":"one"}}]}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeTemplateReadOnly) { + t.Errorf("code = %v, want %s", body["code"], CodeTemplateReadOnly) + } + if body["item_index"] != float64(1) || body["item_kind"] != "update" || body["item_issue_id"] != "bd-1" { + t.Errorf("item members = %v/%v/%v, want 1/update/bd-1", body["item_index"], body["item_kind"], body["item_issue_id"]) + } + assertTemplateDetailIsOurs(t, body["detail"]) +} + +// TestUpdateCarriesAllowTemplateToTheRole pins that `allow_template` is decoded +// onto the role request and nothing else: the role owns what it waives. +func TestUpdateCarriesAllowTemplateToTheRole(t *testing.T) { + for _, tc := range []struct { + body string + want bool + }{ + {`{"actor":"alice","patch":{"title":"edited"},"allow_template":true}`, true}, + {`{"actor":"alice","patch":{"title":"edited"},"allow_template":false}`, false}, + {`{"actor":"alice","patch":{"title":"edited"}}`, false}, + } { + lifecycle := &roleLifecycle{updateResult: issueops.UpdateResult{Issue: updatedIssue("bd-1"), Changed: true}} + ts := newUpdateServer(t, lifecycle) + resp := ts.updateIssue(t, updatePath, tc.body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s: status = %d, want 200: %s", tc.body, resp.StatusCode, readAll(t, resp)) + } + _ = resp.Body.Close() + got := lifecycle.updateRequests() + if len(got) != 1 || got[0].AllowTemplate != tc.want { + t.Errorf("%s: the role received %+v, want AllowTemplate=%v", tc.body, got, tc.want) + } + } +} + +func TestUpdateRefusesANonBooleanAllowTemplate(t *testing.T) { + lifecycle := &roleLifecycle{} + ts := newUpdateServer(t, lifecycle) + resp := ts.updateIssue(t, updatePath, `{"actor":"alice","patch":{"title":"edited"},"allow_template":"yes"}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + if got := lifecycle.updateRequests(); len(got) != 0 { + t.Errorf("the role was reached with %+v", got) + } +} diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index a9c50dec3..89c866bab 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -4781,6 +4781,12 @@ "type": "string", "required": true }, + { + "schema": "UpdateIssueRequest", + "member": "allow_template", + "type": "boolean", + "required": false + }, { "schema": "UpdateIssueRequest", "member": "claim", diff --git a/internal/httpclient/lifecycle.go b/internal/httpclient/lifecycle.go index 06f583730..1bad6d62c 100644 --- a/internal/httpclient/lifecycle.go +++ b/internal/httpclient/lifecycle.go @@ -15,6 +15,7 @@ import ( "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpclient/encode" "github.com/steveyegge/beads/internal/httpclient/wire" + storageops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -330,6 +331,11 @@ func (l *httpLifecycle) Update(ctx context.Context, req issueops.UpdateRequest) return issueops.UpdateResult{}, invalid("update names no field to write") } + req, err = l.applyTemplateGuardForServer(ctx, req) + if err != nil { + return issueops.UpdateResult{}, err + } + res, err := l.wire.UpdateIssue(ctx, req.IssueID, req.Actor, patch, updateGuards(req), updateFlags(req)) if err != nil { if req.Claim && isClaimOnlyUpdate(req) && serverPredatesUpdateClaim(err) { @@ -384,6 +390,7 @@ func updateFlags(req issueops.UpdateRequest) wire.UpdateFlags { ForceAssigneeTransfer: req.ForceAssigneeTransfer, ForceClosePolicy: req.ForceClosePolicy, ForceNotesOverwrite: req.ForceNotesOverwrite, + AllowTemplate: req.AllowTemplate, } } @@ -406,6 +413,38 @@ func serverPredatesUpdateClaim(err error) bool { problem.Param == "claim" } +// applyTemplateGuardForServer decides, BEFORE the dial and from the cached +// handshake, how the template read-only guard (storage +// issueops.AuthorizeTemplateUpdate) reaches the server this request goes to. +// +// A server advertising wire.CapIssuesUpdateAllowTemplate enforces the guard in +// the update's own transaction and accepts `allow_template`, so req goes out +// as is. A server without the token predates both: it would edit a template +// for any caller, and it refuses `allow_template` as an unknown parameter. So +// against it the client runs the same rule itself, on a pre-read of the row — +// the refusal `bd update` and `bd assign` always made before dialing — and +// clears AllowTemplate, which that server has no guard to stand down. A row +// the pre-read cannot find goes out unchanged: the server's own not-found +// answer is the one to report. +func (l *httpLifecycle) applyTemplateGuardForServer(ctx context.Context, req issueops.UpdateRequest) (issueops.UpdateRequest, error) { + snap, err := l.store.snapshot(ctx) + if err != nil { + return req, err + } + if snap != nil && slices.Contains(snap.Capabilities, wire.CapIssuesUpdateAllowTemplate) { + return req, nil + } + if req.AllowTemplate { + req.AllowTemplate = false + return req, nil + } + before, err := l.store.GetIssue(ctx, req.IssueID) + if err != nil { + return req, err + } + return req, storageops.AuthorizeTemplateUpdate(before, req) +} + // Close dials POST issues/{id}:close. func (l *httpLifecycle) Close(ctx context.Context, req issueops.CloseRequest) (issueops.CloseResult, error) { if err := requireActor(req.Actor); err != nil { @@ -625,6 +664,7 @@ func isClaimOnlyUpdate(req issueops.UpdateRequest) bool { !req.ForceAssigneeTransfer && !req.ForceClosePolicy && !req.ForceNotesOverwrite && + !req.AllowTemplate && !req.IssuePlaneOnly && req.Provenance == "" && req.ExpectedVersion == nil && diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index d7fa20172..fa21e6b0c 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -247,6 +247,20 @@ func TestServedBatchApplyRefusesAnUnusableRequest(t *testing.T) { conformance.RunBatchApplyRefusesAnUnusableRequest(t, t.Context(), newServedBatchApplyFixture(t, "hba26")) } +func TestServedBatchApplyUpdateItemsRefuseATemplate(t *testing.T) { + conformance.RunBatchApplyUpdateItemsRefuseATemplate(t, t.Context(), newServedBatchApplyFixture(t, "hbatu")) +} + +// TestServedBatchApplySplicesTheMetadataOfATemplateItCreates is PARKED: a create +// item cannot make a template over the v0 wire (ApplyCreateItem publishes no +// is_template member), so the client refuses the plan before the splice could +// be reached. The served update guard itself is pinned by the case above. +func TestServedBatchApplySplicesTheMetadataOfATemplateItCreates(t *testing.T) { + skipKnownDivergence(t, "W-CreateItem.Issue", parkBead, + "ApplyCreateItem publishes no is_template member, so a plan that creates a template is refused client-side") + conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, t.Context(), newServedBatchApplyFixture(t, "hbats")) +} + // TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed wires the dep_add // HasSpawner/ThreadID lineage conformance case onto the http leg (S5): a // served default Config always advertises CapBatchApplyDepAddLineage (it is diff --git a/internal/httpclient/served_claim_lifecycle_test.go b/internal/httpclient/served_claim_lifecycle_test.go index 77a995752..9f8033046 100644 --- a/internal/httpclient/served_claim_lifecycle_test.go +++ b/internal/httpclient/served_claim_lifecycle_test.go @@ -762,6 +762,19 @@ func TestServedLifecycleUpdateProvenanceLabelsHistory(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, t.Context(), newServedUpdateFixture(t, "hlpv")) } +// TestServedLifecycleUpdateRefusesATemplate is the served leg of the template +// guard: bd serve refuses with template_read_only and the client rebuilds the +// typed error — sentence included — the embedded store returns. +func TestServedLifecycleUpdateRefusesATemplate(t *testing.T) { + conformance.RunLifecycleUpdateRefusesATemplate(t, t.Context(), newServedUpdateFixture(t, "hlut")) +} + +// TestServedLifecycleUpdateAllowTemplateEditsATemplate is the served leg of +// the guard's stand-down: allow_template rides the wire and the role honours it. +func TestServedLifecycleUpdateAllowTemplateEditsATemplate(t *testing.T) { + conformance.RunLifecycleUpdateAllowTemplateEditsATemplate(t, t.Context(), newServedUpdateFixture(t, "hlua")) +} + func TestServedLifecycleUpdateRefusesUnknownIDsAndActorlessRequests(t *testing.T) { conformance.RunLifecycleUpdateRefusesUnknownIDsAndActorlessRequests(t, t.Context(), newServedUpdateFixture(t, "hluu")) } diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 60476683a..a52f81cb1 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -209,6 +209,18 @@ const CapSweepLimit = "issues.sweep.limit" // asked to carry against an older server. const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" +// CapIssuesUpdateAllowTemplate is the behavior capability announcing that +// updateIssue enforces the template read-only guard and accepts +// `allow_template` to stand it down, spelled exactly as httpapi's constant of +// the same name. Lifecycle.Update reads it from the cached handshake before +// the dial (applyTemplateGuardForServer): against a server without it, which +// predates the guard, the client refuses a template update itself on a +// pre-read and never sends `allow_template`. BatchApplier does not read it: +// an applyBatch `update` item gets no client-side check (its target resolves +// on the server), so through an older server it still edits a template +// (bd-jkp9v3). +const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" + // CapExternalDependencies is the CONDITIONAL behavior capability announcing // that the ready, claim and close operations of this server apply bd's // external-dependency policy themselves, spelled exactly as httpapi's constant @@ -349,7 +361,7 @@ func (e *WireRevisionSkewError) Unwrap() error { return ErrWireRevisionSkew } var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, - CapBatchApplyDepAddLineage, + CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/problem.go b/internal/httpclient/wire/problem.go index db2371718..a02dc3d34 100644 --- a/internal/httpclient/wire/problem.go +++ b/internal/httpclient/wire/problem.go @@ -638,6 +638,13 @@ var codeSentinel = map[string]func(*ProblemError, target) error{ // sentinel callers already switch on locally.] "notes_overwrite_refused": func(*ProblemError, target) error { return issueops.ErrNotesOverwrite }, + // The template guard, rebuilt whole so a caller reads the same typed error + // — and the same message — the embedded store returns. The row comes from + // the request or, on a batch, from `item_issue_id`. + "template_read_only": func(e *ProblemError, t target) error { + return &issueops.TemplateReadOnlyError{IssueID: refusedIssueID(e, t)} + }, + // The same rule again: `issue_id` present means the edge named the issue's // own ancestor or descendant, absent means a plain scheduling cycle. The // three hierarchy members travel together and rebuild the typed error whole diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index c650ac7f5..01a0ea9cf 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -196,11 +196,11 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if !slices.Equal(behaviorCapabilities, []string{ httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, - httpapi.CapBatchApplyDepAddLineage, + httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit and "+ - "issues.batchApply.depAddLineage tokens", behaviorCapabilities) + "issues.batchApply.depAddLineage and issues.update.allowTemplate tokens", behaviorCapabilities) } } diff --git a/internal/httpclient/wire/writes.go b/internal/httpclient/wire/writes.go index a6791100f..c9557f1b1 100644 --- a/internal/httpclient/wire/writes.go +++ b/internal/httpclient/wire/writes.go @@ -228,7 +228,7 @@ type updateBody struct { Actor string `json:"actor"` Patch map[string]any `json:"patch"` UpdateGuards - // The four UpdateFlags, each sent only when true (setItemBool's + // The five UpdateFlags, each sent only when true (setItemBool's // convention: an explicit false is the default said twice). Absent is // what an older server that predates a member reads correctly, which is // what lets a request that sets none of them reach a server that knows @@ -237,15 +237,16 @@ type updateBody struct { ForceAssigneeTransfer *bool `json:"force_assignee_transfer,omitempty"` ForceClosePolicy *bool `json:"force_close_policy,omitempty"` ForceNotesOverwrite *bool `json:"force_notes_overwrite,omitempty"` + AllowTemplate *bool `json:"allow_template,omitempty"` } -// UpdateFlags are updateIssue's four top-level booleans: the claim, and the -// three force overrides. UpdateIssueRequest publishes all four -// (internal/httpapi/apigen's generated type) and none is a GUARD — none carries -// a comparison value, unlike every member UpdateGuards holds — so they travel -// as their own struct rather than joining that one. +// UpdateFlags are updateIssue's five top-level booleans: the claim, the three +// force overrides and the template stand-down. UpdateIssueRequest publishes all +// five (internal/httpapi/apigen's generated type) and none is a GUARD — none +// carries a comparison value, unlike every member UpdateGuards holds — so they +// travel as their own struct rather than joining that one. // -// It is a struct rather than four arguments for UpdateGuards' reason: four +// It is a struct rather than five arguments for UpdateGuards' reason: five // positional booleans side by side are the shape a caller transposes, and a // transposed force flag is a bypass nobody asked for. type UpdateFlags struct { @@ -268,6 +269,9 @@ type UpdateFlags struct { // replace existing non-empty notes with different non-empty content. The // server requires patch.notes beside it. ForceNotesOverwrite bool + // AllowTemplate stands the template read-only refusal down for this + // request (bd label, bd set-state edit templates by design). + AllowTemplate bool } // UpdateIssue patches one issue. It is the only PATCH on this surface: the @@ -291,6 +295,7 @@ func (c *Client) UpdateIssue(ctx context.Context, id, actor string, patch map[st ForceAssigneeTransfer: trueOrAbsent(flags.ForceAssigneeTransfer), ForceClosePolicy: trueOrAbsent(flags.ForceClosePolicy), ForceNotesOverwrite: trueOrAbsent(flags.ForceNotesOverwrite), + AllowTemplate: trueOrAbsent(flags.AllowTemplate), } r := Request{Op: OpUpdateIssue, Method: http.MethodPatch, Path: path, Body: body, IssueID: id} if err := c.dispatch(ctx, r, &out); err != nil { diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go index 945c51477..69f3514cd 100644 --- a/internal/httpclient/write_ledger_test.go +++ b/internal/httpclient/write_ledger_test.go @@ -262,6 +262,8 @@ func writeShapes() []writeShape { "ForceAssigneeTransfer": member("force_assignee_transfer"), "ForceClosePolicy": member("force_close_policy"), "Claim": member("claim"), + // The template guard's stand-down (bd label, bd set-state). + "AllowTemplate": member("allow_template"), }, }, { diff --git a/internal/httpclient/write_roles_test.go b/internal/httpclient/write_roles_test.go index e58916379..a4456dab6 100644 --- a/internal/httpclient/write_roles_test.go +++ b/internal/httpclient/write_roles_test.go @@ -383,7 +383,13 @@ func (s *stubWire) ListReadyWork(_ context.Context, _ url.Values) (*apigen.Ready func stubStore(t *testing.T, w *stubWire) *Store { t.Helper() - return New(testTarget(t), w, &apigen.ContextResponse{BdVersion: "1.2.3"}) + // A current server: it advertises issues.update.allowTemplate, so + // Lifecycle.Update leaves the template guard to it and pre-reads nothing. + // TestUpdateTemplateGuardAgainstAServerThatPredatesIt covers the older one. + return New(testTarget(t), w, &apigen.ContextResponse{ + BdVersion: "1.2.3", + Capabilities: []string{wire.CapIssuesUpdateAllowTemplate}, + }) } func set[T any](v T) issueops.Field[T] { return issueops.Field[T]{Set: true, Value: v} } @@ -951,6 +957,12 @@ func TestUpdateSendsEachFlagOnlyWhenRequested(t *testing.T) { want: wire.UpdateFlags{ForceClosePolicy: true}, wantPatch: map[string]any{"status": string(issueops.StatusClosed)}, }, + { + name: "allow template", + req: issueops.UpdateRequest{Patch: notes, AllowTemplate: true}, + want: wire.UpdateFlags{AllowTemplate: true}, + wantPatch: map[string]any{"notes": "replacement notes"}, + }, { name: "a claim alone sends the empty patch", req: issueops.UpdateRequest{Claim: true}, @@ -1812,3 +1824,89 @@ func TestWriteAccessorsRefuseWithoutATransport(t *testing.T) { t.Errorf("CloseIssue without a transport = %v, want ErrNoTransport", err) } } + +// templateServerWire is a stubWire whose generic read half answers getIssue +// with row (nil: not found), recording the read in the same call log as the +// writes, so a test can see whether a pre-read ran and whether it ran BEFORE +// updateIssue — or instead of it. +type templateServerWire struct { + *stubWire + row *types.Issue +} + +func (w *templateServerWire) Preflight(context.Context, string) error { return nil } + +func (w *templateServerWire) Do(_ context.Context, req wire.Request, out any) error { + if req.Op != wire.OpGetIssue { + return fmt.Errorf("templateServerWire: unexpected generic dispatch %q", req.Op) + } + w.calls = append(w.calls, "getIssue:"+req.IssueID) + if w.row == nil { + return fmt.Errorf("no issue %s: %w", req.IssueID, issueops.ErrNotFound) + } + *out.(*types.IssueDetails) = types.IssueDetails{Issue: *w.row, Revision: "1"} + return nil +} + +// TestUpdateTemplateGuardAgainstAServerThatPredatesIt is a NEW client against an +// OLD server: a handshake without issues.update.allowTemplate is a server that +// applies no template guard and refuses `allow_template` as unknown. The client +// decides from the cached handshake, before the dial: it refuses a template +// update itself on a pre-read (the refusal bd update always made), never sends +// `allow_template` to it, and against a server that DOES advertise the token +// it neither pre-reads nor drops the member. +func TestUpdateTemplateGuardAgainstAServerThatPredatesIt(t *testing.T) { + template := &types.Issue{ID: "bd-1", Title: "tmpl", IsTemplate: true} + plain := &types.Issue{ID: "bd-1", Title: "work"} + for _, tc := range []struct { + name string + guarded bool // the handshake advertises the token + row *types.Issue + allow bool + wantCalls []string + wantRefused bool + wantFlag bool + }{ + {name: "old server: a template update is refused before the dial", row: template, + wantCalls: []string{"getIssue:bd-1"}, wantRefused: true}, + {name: "old server: a plain row goes out after the pre-read", row: plain, + wantCalls: []string{"getIssue:bd-1", "updateIssue:bd-1"}}, + {name: "old server: a row the pre-read cannot find goes out for the server to answer", row: nil, + wantCalls: []string{"getIssue:bd-1", "updateIssue:bd-1"}}, + {name: "old server: allow_template is not sent and no pre-read runs", row: template, allow: true, + wantCalls: []string{"updateIssue:bd-1"}}, + {name: "guarded server: no pre-read, the server owns the refusal", guarded: true, row: template, + wantCalls: []string{"updateIssue:bd-1"}}, + {name: "guarded server: allow_template rides the wire", guarded: true, row: template, allow: true, + wantCalls: []string{"updateIssue:bd-1"}, wantFlag: true}, + } { + t.Run(tc.name, func(t *testing.T) { + w := &templateServerWire{stubWire: &stubWire{update: &apigen.UpdateIssueResponse{Revision: "3"}}, row: tc.row} + snap := &apigen.ContextResponse{BdVersion: "1.2.3"} + if tc.guarded { + snap.Capabilities = []string{wire.CapIssuesUpdateAllowTemplate} + } + lifecycle, err := New(testTarget(t), w, snap).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + _, err = lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "writer", IssueID: "bd-1", AllowTemplate: tc.allow, + Patch: issueops.IssuePatch{Title: set("t")}, + }) + var refusal *issueops.TemplateReadOnlyError + if got := errors.As(err, &refusal); got != tc.wantRefused { + t.Fatalf("Update err = %v, want a TemplateReadOnlyError: %v", err, tc.wantRefused) + } + if !tc.wantRefused && err != nil { + t.Fatalf("Update: %v", err) + } + if !reflect.DeepEqual(w.calls, tc.wantCalls) { + t.Errorf("calls = %v, want %v", w.calls, tc.wantCalls) + } + if !tc.wantRefused && w.lastFlags.AllowTemplate != tc.wantFlag { + t.Errorf("sent allow_template = %v, want %v", w.lastFlags.AllowTemplate, tc.wantFlag) + } + }) + } +} diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index 13b950360..480150703 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -255,6 +255,16 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplyRefusesAnUnusableRequest(t, ctx, fixture) }) + t.Run("UpdateItemsRefuseATemplate", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyUpdateItemsRefuseATemplate(t, ctx, fixture) + }) + t.Run("SplicesTheMetadataOfATemplateItCreates", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) + }) } // newDoltBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/dolt/lifecycle_update_contract_test.go b/internal/storage/dolt/lifecycle_update_contract_test.go index 409ad0b3f..2696c7834 100644 --- a/internal/storage/dolt/lifecycle_update_contract_test.go +++ b/internal/storage/dolt/lifecycle_update_contract_test.go @@ -90,6 +90,12 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("ProvenanceLabelsHistory", func(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, ctx, fixture) }) + t.Run("RefusesATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateRefusesATemplate(t, ctx, fixture) + }) + t.Run("AllowTemplateEditsATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateAllowTemplateEditsATemplate(t, ctx, fixture) + }) } func newDoltLifecycleUpdateFixture(t *testing.T, prefix string) (conformance.LifecycleUpdateFixture, context.Context, func()) { diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index c7be4ca2d..5bf1afbaa 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -187,6 +187,12 @@ func TestBatchApplyContract(t *testing.T) { t.Run("RefusesAnUnusableRequest", func(t *testing.T) { conformance.RunBatchApplyRefusesAnUnusableRequest(t, ctx, fixture) }) + t.Run("UpdateItemsRefuseATemplate", func(t *testing.T) { + conformance.RunBatchApplyUpdateItemsRefuseATemplate(t, ctx, fixture) + }) + t.Run("SplicesTheMetadataOfATemplateItCreates", func(t *testing.T) { + conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) + }) } // newEmbeddedBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/embeddeddolt/lifecycle_update_contract_test.go b/internal/storage/embeddeddolt/lifecycle_update_contract_test.go index 985ff24f9..635c3ff13 100644 --- a/internal/storage/embeddeddolt/lifecycle_update_contract_test.go +++ b/internal/storage/embeddeddolt/lifecycle_update_contract_test.go @@ -91,6 +91,12 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("ProvenanceLabelsHistory", func(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, ctx, fixture) }) + t.Run("RefusesATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateRefusesATemplate(t, ctx, fixture) + }) + t.Run("AllowTemplateEditsATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateAllowTemplateEditsATemplate(t, ctx, fixture) + }) } func newEmbeddedLifecycleUpdateFixture(t *testing.T, te *testEnv, prefix string) conformance.LifecycleUpdateFixture { diff --git a/internal/storage/issueops/aggregate.go b/internal/storage/issueops/aggregate.go index f56fddc04..faf899e7c 100644 --- a/internal/storage/issueops/aggregate.go +++ b/internal/storage/issueops/aggregate.go @@ -231,6 +231,29 @@ func AuthorizeNotesOverwrite(before *types.Issue, request publicops.UpdateReques return fmt.Errorf("%w: issue %s", storage.ErrNotesOverwrite, before.ID) } +// AuthorizeTemplateUpdate refuses an update of a template with +// *publicops.TemplateReadOnlyError. Templates are read-only — work comes out of +// one by pouring it — so every update naming one refuses, whatever its patch +// and whatever force flags it carries: no force flag waives it (only the +// caller's explicit AllowTemplate does). +// +// It is the guard `bd update` (and assign, and the proxied mutations) used to +// apply as a pre-read in cmd/bd, which left bd serve, BatchApplier update items +// and library callers writing to templates bd refused. It sits beside the +// other update fences and is called from the same three places — ExecuteUpdate, +// and the unit-of-work update and batch legs — over the same-transaction +// pre-image, after the compare-and-set preconditions, so a stale guard still +// reports as the mismatch. +// +// request.AllowTemplate is the caller saying it edits a template deliberately +// (bd label, bd set-state); the guard stands down for that request only. +func AuthorizeTemplateUpdate(before *types.Issue, request publicops.UpdateRequest) error { + if before != nil && before.IsTemplate && !request.AllowTemplate { + return &publicops.TemplateReadOnlyError{IssueID: before.ID} + } + return nil +} + // ApplyMetadataPatch returns the canonical metadata value and whether it changes. func ApplyMetadataPatch(current json.RawMessage, patch publicops.MetadataPatch) (json.RawMessage, bool, error) { if !patch.Replace.Set && !patch.Merge.Set && len(patch.Set) == 0 && len(patch.Unset) == 0 { diff --git a/internal/storage/issueops/batch_apply.go b/internal/storage/issueops/batch_apply.go index bd6a251d1..1621e772e 100644 --- a/internal/storage/issueops/batch_apply.go +++ b/internal/storage/issueops/batch_apply.go @@ -481,11 +481,13 @@ func (r *applyBatchRun) spliceMetadataRefs(ctx context.Context, tx *sql.Tx) erro return err } id := r.result.Items[index].IssueID - updated, tables, err := ExecuteUpdate(ctx, tx, publicops.UpdateRequest{ + // Unguarded for templates: the splice finishes this request's own + // create, so a template it creates is not being modified. + updated, tables, err := executeUpdate(ctx, tx, publicops.UpdateRequest{ Actor: r.plan.Actor, IssueID: id, Patch: publicops.IssuePatch{Metadata: publicops.MetadataPatch{Set: set}}, - }) + }, false) if err != nil { return &publicops.ItemError{ Index: index, Kind: publicops.ItemCreate, Key: item.Create.Key, IssueID: id, Err: err, diff --git a/internal/storage/issueops/execution.go b/internal/storage/issueops/execution.go index 04009d857..c701fb55f 100644 --- a/internal/storage/issueops/execution.go +++ b/internal/storage/issueops/execution.go @@ -136,6 +136,15 @@ func skippedDependencyError(skipped []skippedDependency) error { // ExecuteUpdate applies a guarded update in tx and reports durable tables changed. func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequest) (publicops.UpdateResult, ChangedTables, error) { + return executeUpdate(ctx, tx, request, true) +} + +// executeUpdate is ExecuteUpdate with the template guard made optional, for the +// one caller that is not an update of an existing row: the batch apply's +// metadata splice, which finishes a CREATE item of the same request — a +// template the request itself is creating is still being written, not +// modified. +func executeUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequest, guardTemplate bool) (publicops.UpdateResult, ChangedTables, error) { attempt := CloneUpdateRequest(request) if attempt.Actor == "" || attempt.IssueID == "" { return publicops.UpdateResult{}, nil, fmt.Errorf("%w: update requires actor and issue ID", storage.ErrValidation) @@ -171,6 +180,11 @@ func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequ } else if err := CheckExpectedFieldsInTx(ctx, tx, attempt.IssueID, attempt.ExpectedAssignee, nil); err != nil { return publicops.UpdateResult{}, nil, err } + if guardTemplate { + if err := AuthorizeTemplateUpdate(before, attempt); err != nil { + return publicops.UpdateResult{}, nil, err + } + } if err := AuthorizeAssigneeTransfer(ctx, tx, before, attempt); err != nil { return publicops.UpdateResult{}, nil, err } diff --git a/internal/storage/issueops/version_completeness_test.go b/internal/storage/issueops/version_completeness_test.go index e7d265b11..611c55c6d 100644 --- a/internal/storage/issueops/version_completeness_test.go +++ b/internal/storage/issueops/version_completeness_test.go @@ -76,8 +76,10 @@ func versionMintEdges(fns map[string]*journalscan.FuncInfo) func(*journalscan.Fu // its own RecordVersionInTx: nothing it calls with the gate off can carry it // on its behalf. TestCompositeMutationsCarryTheirOwnMint is what turns // reviewer Probe 2 (delete ExecuteUpdate's final mint block) into a failure. +// ExecuteUpdate's body is executeUpdate (ExecuteUpdate is it with the template +// guard on), so the composite named here is the body that carries the mint. var versionCompositeMutations = []string{ - "ExecuteUpdate", + "executeUpdate", "applyLabelPatch", "applyParentPatch", } diff --git a/internal/storage/uow/batch_applier.go b/internal/storage/uow/batch_applier.go index 9c41e1ae6..b26025a5c 100644 --- a/internal/storage/uow/batch_applier.go +++ b/internal/storage/uow/batch_applier.go @@ -310,7 +310,7 @@ func (r *uowApplyRun) applyUpdate(ctx context.Context, index int, item *publicop if err := validateUpdateRequest(request); err != nil { return itemErr(err) } - updated, err := r.runUpdate(ctx, request) + updated, err := r.runUpdate(ctx, request, true) if err != nil { return itemErr(err) } @@ -332,7 +332,10 @@ func (r *uowApplyRun) applyUpdate(ctx context.Context, index int, item *publicop // change. ga-v2k49 is a concrete instance of that drift: claimChanged's actor // comparison here and there diverged (one verbatim, one not) until both were // found and fixed independently. -func (r *uowApplyRun) runUpdate(ctx context.Context, request publicops.UpdateRequest) (publicops.UpdateResult, error) { +// +// guardTemplate is false only for the metadata splice, which finishes a create +// item of this request rather than modifying an existing row. +func (r *uowApplyRun) runUpdate(ctx context.Context, request publicops.UpdateRequest, guardTemplate bool) (publicops.UpdateResult, error) { spec, err := updateSpec(request) if err != nil { return publicops.UpdateResult{}, validationError(err) @@ -346,6 +349,11 @@ func (r *uowApplyRun) runUpdate(ctx context.Context, request publicops.UpdateReq return publicops.UpdateResult{}, err } if updatePreconditionsHold(request, before) { + if guardTemplate { + if err := storageissueops.AuthorizeTemplateUpdate(before, request); err != nil { + return publicops.UpdateResult{}, err + } + } if err := authorizeAssigneeTransfer(ctx, r.uw, before, request); err != nil { return publicops.UpdateResult{}, err } @@ -527,11 +535,13 @@ func (r *uowApplyRun) spliceMetadataRefs(ctx context.Context) error { return err } id := r.result.Items[index].IssueID + // Unguarded for templates, as the store-backed splice is: it finishes + // this request's own create. updated, err := r.runUpdate(ctx, publicops.UpdateRequest{ Actor: r.plan.Actor, IssueID: id, Patch: publicops.IssuePatch{Metadata: publicops.MetadataPatch{Set: set}}, - }) + }, false) if err != nil { return &publicops.ItemError{Index: index, Kind: publicops.ItemCreate, Key: item.Create.Key, IssueID: id, Err: err} } diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index 6ae8f3899..9bb82d3e9 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -157,6 +157,12 @@ func TestBatchApplyContract(t *testing.T) { t.Run("RefusesAnUnusableRequest", func(t *testing.T) { conformance.RunBatchApplyRefusesAnUnusableRequest(t, ctx, fixture) }) + t.Run("UpdateItemsRefuseATemplate", func(t *testing.T) { + conformance.RunBatchApplyUpdateItemsRefuseATemplate(t, ctx, fixture) + }) + t.Run("SplicesTheMetadataOfATemplateItCreates", func(t *testing.T) { + conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) + }) } func newUOWBatchApplyFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchApplyFixture { diff --git a/internal/storage/uow/issue_operations.go b/internal/storage/uow/issue_operations.go index c0f3451d8..55205753a 100644 --- a/internal/storage/uow/issue_operations.go +++ b/internal/storage/uow/issue_operations.go @@ -231,6 +231,9 @@ func (o *issueOperations) Update(ctx context.Context, request publicops.UpdateRe // version and status before authorizing, so when one is stale the fence // stands down and ApplyUpdate reports the mismatch instead. if updatePreconditionsHold(attempt, before) { + if err := storageissueops.AuthorizeTemplateUpdate(before, attempt); err != nil { + return publicops.UpdateResult{}, "", err + } if err := authorizeAssigneeTransfer(ctx, uw, before, attempt); err != nil { return publicops.UpdateResult{}, "", err } diff --git a/internal/storage/uow/issue_operations_claim_contract_test.go b/internal/storage/uow/issue_operations_claim_contract_test.go index c4c84067b..9ecc081a2 100644 --- a/internal/storage/uow/issue_operations_claim_contract_test.go +++ b/internal/storage/uow/issue_operations_claim_contract_test.go @@ -289,7 +289,7 @@ func TestBatchRunUpdateIdempotentClaimAcrossSpellingReportsNoChange(t *testing.T result, err := run.runUpdate(context.Background(), publicops.UpdateRequest{ Actor: "gastown.mayor", IssueID: "bd-1", Claim: true, - }) + }, true) if err != nil { t.Fatalf("runUpdate: %v, want success (same identity, different separator spelling)", err) } @@ -307,7 +307,7 @@ func TestBatchRunUpdateClaimByGenuinelyDifferentIdentityReportsAChange(t *testin result, err := run.runUpdate(context.Background(), publicops.UpdateRequest{ Actor: "gastown.dog-3", IssueID: "bd-1", Claim: true, - }) + }, true) if err != nil { t.Fatalf("runUpdate: %v", err) } diff --git a/internal/storage/uow/lifecycle_update_contract_test.go b/internal/storage/uow/lifecycle_update_contract_test.go index 1e7790e39..85983bf4f 100644 --- a/internal/storage/uow/lifecycle_update_contract_test.go +++ b/internal/storage/uow/lifecycle_update_contract_test.go @@ -92,6 +92,12 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("ProvenanceLabelsHistory", func(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, ctx, fixture) }) + t.Run("RefusesATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateRefusesATemplate(t, ctx, fixture) + }) + t.Run("AllowTemplateEditsATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateAllowTemplateEditsATemplate(t, ctx, fixture) + }) } func newUOWLifecycleUpdateFixture(t *testing.T, ctx context.Context, prefix string) conformance.LifecycleUpdateFixture { diff --git a/internal/validation/BUILD.bazel b/internal/validation/BUILD.bazel index 5b75f8787..e9aefda09 100644 --- a/internal/validation/BUILD.bazel +++ b/internal/validation/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "//internal/config", "//internal/types", "//internal/utils", + "//issueops", ], ) diff --git a/internal/validation/issue.go b/internal/validation/issue.go index c59f4e4b2..bde847ec3 100644 --- a/internal/validation/issue.go +++ b/internal/validation/issue.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) // IssueValidator validates an issue and returns an error if validation fails. @@ -43,7 +44,7 @@ func NotTemplate() IssueValidator { return nil // Let Exists() handle nil check if needed } if issue.IsTemplate { - return fmt.Errorf("cannot modify template %s: templates are read-only; use 'bd mol pour' to create a work item", id) + return &issueops.TemplateReadOnlyError{IssueID: id} } return nil } diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index 47fa9a549..817b49916 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -33,6 +33,7 @@ go_library( "releaser.go", "statsreporter.go", "sweeper.go", + "template_errors.go", "treewalker.go", "versionreconciler.go", "workspaceconfig.go", diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index ac84c74df..76d301234 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -104,7 +104,9 @@ type CreateItem struct { // UpdateItem patches one existing issue, under UpdateRequest's rules. type UpdateItem struct { - // Target names the issue to patch and must resolve BACKWARD — see Ref. + // Target names the issue to patch and must resolve BACKWARD — see Ref. A + // template target refuses with *TemplateReadOnlyError, exactly as + // Lifecycle.Update does: templates are read-only. Target Ref // Patch is the edit, read exactly as UpdateRequest.Patch is — including // that Patch.Labels is the whole LabelPatch, so a label REMOVAL is diff --git a/issueops/issueops.go b/issueops/issueops.go index aa2bd2101..69c8a20a1 100644 --- a/issueops/issueops.go +++ b/issueops/issueops.go @@ -273,6 +273,14 @@ type UpdateRequest struct { // It is the update-side spelling of CloseRequest.Force; a command adapter // that maps one flag to both spells both. ForceClosePolicy bool + // AllowTemplate says the caller edits a template deliberately, so the + // template read-only refusal (*TemplateReadOnlyError) stands down for this + // request. bd label and bd set-state set it to keep their pre-guard + // behavior: they have always written to templates. The zero value + // enforces the guard. It bypasses nothing else — validation, the + // preconditions, the assignee and notes fences and close policy still + // apply — and has no effect on an issue that is not a template. + AllowTemplate bool // The three Expected* guards below are this package's FOUNDING spelling of // the compare-and-set family, and the family's rules are stated once, at // length, on DeleteRequest.ExpectedVersion (deleter.go) and @@ -478,7 +486,12 @@ type Lifecycle interface { // returns ErrCloseBlocked, both without mutation. ForceClosePolicy bypasses // those two refusals and nothing else. A Claim that loses its // compare-and-set returns *ClaimConflictError carrying the state that beat - // it. A refusal or validation error leaves persistent state unchanged. + // it. An update of a template returns *TemplateReadOnlyError whatever its + // patch and force flags, unless it sets AllowTemplate — templates are + // read-only — checked after the + // compare-and-set preconditions, so a stale guard still reports the + // mismatch. A refusal or validation error leaves persistent state + // unchanged. Update(context.Context, UpdateRequest) (UpdateResult, error) // Close validates guards and commits the complete request as one atomic // mutation. It moves the issue to literal StatusClosed, including from a diff --git a/issueops/template_errors.go b/issueops/template_errors.go new file mode 100644 index 000000000..8997324c3 --- /dev/null +++ b/issueops/template_errors.go @@ -0,0 +1,30 @@ +package issueops + +import ( + "errors" + "fmt" +) + +// ErrTemplateReadOnly is returned when a mutation that guards templates names +// one. Every update does (Lifecycle.Update, a BatchApplier update item); not +// every verb does yet (bd-jkp9v3). Templates are read-only: the way to get +// work out of one is to pour it (`bd mol pour`), which creates new issues and +// leaves the template untouched. There is no force bypass — no Force flag +// waives it; only an update that sets UpdateRequest.AllowTemplate (bd label, +// bd set-state) is let through. +var ErrTemplateReadOnly = errors.New("templates are read-only") + +// TemplateReadOnlyError reports the template a mutation was refused for. Its +// message is the sentence `bd` has always printed for the refusal, so a caller +// that renders err.Error() reads the same line whichever backend refused. +type TemplateReadOnlyError struct { + // IssueID names the template that refused the mutation. + IssueID string +} + +func (e *TemplateReadOnlyError) Error() string { + return fmt.Sprintf("cannot modify template %s: templates are read-only; use 'bd mol pour' to create a work item", e.IssueID) +} + +// Unwrap makes TemplateReadOnlyError match ErrTemplateReadOnly. +func (e *TemplateReadOnlyError) Unwrap() error { return ErrTemplateReadOnly } From a945eca145bd91431e05cf14149b09c9e30bf263 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 17:47:17 +0000 Subject: [PATCH 017/151] ci(bazel): fetch pinned Ubuntu debs in parallel from a gascity mirror first (#7434) * ci(bazel): fetch pinned Ubuntu debs in parallel from a gascity mirror first cc_sysroot_noble_amd64 downloaded its 7 pinned .debs one at a time from snapshot.ubuntu.com, with only archive.ubuntu.com behind it. On 2026-10-09 the snapshot returned 503, and Bazel's retries against it cost ~95 s per cold fetch before falling back (and archive.ubuntu.com drops superseded versions, which would fail the fetch outright). Same change as gastownhall/gascity (ga-vnycm2.11): deb_sysroot takes URL templates ({path}, {file}, {sha256}) and starts every package download at once (rctx.download block = False). NOBLE_DEB_URLS tries gascity's content-addressed release mirror (toolchain-noble-debs-20261001, which holds these exact pins; beads already takes its LLVM slice from a gascity release), the snapshot, Launchpad's librarian, then archive.ubuntu.com. The sha256 pins are unchanged and decide what is accepted; the unpacked sysroot is byte-identical. The hermetic-cc guard now checks every deb_sysroot: placeholders, the snapshot kept, two or more hosts, no moving mirror first, no ICU. Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): auto-sync BUILD files Applied from the bazel-sync-patch artifact of https://github.com/gastownhall/beads/actions/runs/37966817176 (`make bazel-sync`: gazelle, tools/bazel/go_srcs.py, bazel mod tidy). Only BUILD.bazel, MODULE.bazel and MODULE.bazel.lock are ever pushed; see scripts/bazel-autofix-push.sh. --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- MODULE.bazel | 23 +++- scripts/bazel_hermetic_cc_test.go | 136 ++++++++++++++++++----- third_party/cc_toolchain/deb_sysroot.bzl | 35 ++++-- 3 files changed, 149 insertions(+), 45 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 68ded81bb..0be68a91d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -39,12 +39,26 @@ bazel_dep(name = "rules_cc", version = "0.2.17") # scripts/bazel_hermetic_cc_test.go guards both. deb_sysroot = use_repo_rule("//third_party/cc_toolchain:deb_sysroot.bzl", "deb_sysroot") +# Where every pinned Ubuntu 24.04 package below comes from, tried in order +# (the sha256 pins, not the source, decide what is accepted): +# - gascity's content-addressed mirror, a release holding each pinned +# .deb as .deb (gascity's tools/cc_toolchain/mirror_debs.sh +# publishes it; gascity pins the same packages), so a cold fetch does not +# depend on Canonical's servers; +# - the snapshot these packages were pinned from (immutable, but a single +# host that returns 503 for minutes at a time); +# - Launchpad's librarian, which keeps every file ever published to the +# primary archive, superseded versions included; +# - archive.ubuntu.com, which drops superseded versions. +NOBLE_DEB_URLS = [ + "https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb", + "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/{path}", + "https://launchpad.net/ubuntu/+archive/primary/+files/{file}", + "https://archive.ubuntu.com/ubuntu/{path}", +] + deb_sysroot( name = "cc_sysroot_noble_amd64", - mirrors = [ - "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/", - "https://archive.ubuntu.com/ubuntu/", - ], packages = { "pool/main/g/gcc-13/libgcc-13-dev_13.3.0-6ubuntu2~24.04.1_amd64.deb": "cd689db2691edaa10f37329307292796bb599e722e0505c79e14caaa1fe9a93a", "pool/main/g/gcc-13/libstdc++-13-dev_13.3.0-6ubuntu2~24.04.1_amd64.deb": "ee5633e863e19c3381ed97842ce35ed32ede96a3d1ae4e94c051d3036fe21347", @@ -70,6 +84,7 @@ deb_sysroot( "usr/lib/gcc/x86_64-linux-gnu/13/libquadmath*", "usr/lib/gcc/x86_64-linux-gnu/13/libstdc++*.a", ], + urls = NOBLE_DEB_URLS, ) LLVM_VERSION = "22.1.8" diff --git a/scripts/bazel_hermetic_cc_test.go b/scripts/bazel_hermetic_cc_test.go index 335d0e648..fec515bff 100644 --- a/scripts/bazel_hermetic_cc_test.go +++ b/scripts/bazel_hermetic_cc_test.go @@ -33,9 +33,14 @@ const ( var ( sha256HexRE = regexp.MustCompile(`^[0-9a-f]{64}$`) - // snapshot.ubuntu.com serves each timestamp's archive state forever; a - // moving mirror (archive.ubuntu.com) may only follow it as a fallback. - ubuntuSnapshotRE = regexp.MustCompile(`^https://snapshot\.ubuntu\.com/ubuntu/\d{8}T\d{6}Z/$`) + // snapshot.ubuntu.com serves each timestamp's archive state forever, so + // the package pins stay resolvable at the snapshot they were taken from. + ubuntuSnapshotRE = regexp.MustCompile(`^https://snapshot\.ubuntu\.com/ubuntu/\d{8}T\d{6}Z/\{path\}$`) + // archive.ubuntu.com drops superseded versions: a fallback, never first. + ubuntuMovingMirrorRE = regexp.MustCompile(`^https?://([a-z0-9-]+\.)*archive\.ubuntu\.com/`) + // deb_sysroot's URL templates: {path} (pool path), {file} (its basename) + // and {sha256} (the pin) are the only placeholders. + debURLPlaceholderRE = regexp.MustCompile(`\{[^}]*\}`) // A sliced llvm_dist archive is a repack (repack_llvm.sh), not an // upstream release, so it may only come from a gastownhall release: // beads' own mirror or gascity's original asset (same bytes, same sha256). @@ -114,36 +119,96 @@ func checkHermeticCCModule(module string) []error { errs = append(errs, errors.New("llvm_dist must pin its archive by sha256")) } - if sysroot, err := moduleCall(module, "deb_sysroot", `name = "cc_sysroot_noble_amd64"`); err != nil { + if _, err := moduleCall(module, "deb_sysroot", `name = "cc_sysroot_noble_amd64"`); err != nil { errs = append(errs, err) + } + for _, m := range regexp.MustCompile(`(?ms)^deb_sysroot\(\n(.*?)^\)`).FindAllStringSubmatch(module, -1) { + errs = append(errs, checkDebSysroot(module, m[1])...) + } + return errs +} + +// checkDebSysroot: every package is an amd64 pool .deb pinned by sha256 that +// ships no ICU, and its URL templates give each package more than one host, +// keep the snapshot the pins were taken from, and never start with a moving +// mirror. +func checkDebSysroot(module, call string) []error { + var errs []error + name := "deb_sysroot" + if m := regexp.MustCompile(`name = "([^"]+)"`).FindStringSubmatch(call); m != nil { + name += " " + m[1] + } + urls, err := debSysrootURLs(module, call) + if err != nil { + errs = append(errs, errors.New(name+": "+err.Error())) } else { - mirrors := regexp.MustCompile(`(?s)mirrors = \[\s*"([^"]+)"`).FindStringSubmatch(sysroot) - if mirrors == nil || !ubuntuSnapshotRE.MatchString(mirrors[1]) { - errs = append(errs, errors.New("deb_sysroot's first mirror must be an immutable snapshot.ubuntu.com/ubuntu// root")) + hosts := map[string]bool{} + snapshot := false + for _, u := range urls { + holes := debURLPlaceholderRE.FindAllString(u, -1) + if len(holes) != 1 || (holes[0] != "{path}" && holes[0] != "{file}" && holes[0] != "{sha256}") { + errs = append(errs, errors.New(name+": url "+u+" must name the package by exactly one of {path}, {file} or {sha256}")) + } + snapshot = snapshot || ubuntuSnapshotRE.MatchString(u) + host, _, _ := strings.Cut(strings.TrimPrefix(u, "https://"), "/") + hosts[host] = true } - body := regexp.MustCompile(`(?s)packages = \{(.*?)\}`).FindStringSubmatch(sysroot) - if body == nil { - errs = append(errs, errors.New("deb_sysroot has no packages")) - } else { - pkgs := quotedPairs(body[1]) - if len(pkgs) == 0 { - errs = append(errs, errors.New("deb_sysroot has no packages")) - } - for path, sum := range pkgs { - if !strings.HasPrefix(path, "pool/") || !strings.HasSuffix(path, "_amd64.deb") || !sha256HexRE.MatchString(sum) { - errs = append(errs, errors.New("deb_sysroot package "+path+" must be an amd64 pool .deb pinned by sha256")) - } - // ICU policy (engdocs/ICU-POLICY.md): nothing links libicu, - // so the sysroot offers no ICU to link against. - if strings.Contains(path, "/icu/") { - errs = append(errs, errors.New("deb_sysroot package "+path+" ships ICU, which beads must never link (engdocs/ICU-POLICY.md)")) - } - } + if !snapshot { + errs = append(errs, errors.New(name+": urls must include an immutable https://snapshot.ubuntu.com/ubuntu//{path}")) + } + if len(hosts) < 2 { + errs = append(errs, errors.New(name+": urls must span at least two hosts, so one outage cannot fail a cold fetch")) + } + if len(urls) > 0 && ubuntuMovingMirrorRE.MatchString(urls[0]) { + errs = append(errs, errors.New(name+": the first url must not be a moving mirror (archive.ubuntu.com drops superseded packages)")) + } + } + body := regexp.MustCompile(`(?s)packages = \{(.*?)\}`).FindStringSubmatch(call) + if body == nil { + return append(errs, errors.New(name+" has no packages")) + } + pkgs := quotedPairs(body[1]) + if len(pkgs) == 0 { + errs = append(errs, errors.New(name+" has no packages")) + } + for path, sum := range pkgs { + if !strings.HasPrefix(path, "pool/") || !strings.HasSuffix(path, "_amd64.deb") || !sha256HexRE.MatchString(sum) { + errs = append(errs, errors.New(name+" package "+path+" must be an amd64 pool .deb pinned by sha256")) + } + // ICU policy (engdocs/ICU-POLICY.md): nothing links libicu, + // so the sysroot offers no ICU to link against. + if strings.Contains(path, "/icu/") { + errs = append(errs, errors.New(name+" package "+path+" ships ICU, which beads must never link (engdocs/ICU-POLICY.md)")) } } return errs } +// debSysrootURLs returns a deb_sysroot call's url templates: an inline list, +// or a top-level MODULE.bazel list constant. +func debSysrootURLs(module, call string) ([]string, error) { + m := regexp.MustCompile(`(?s)urls = (\[.*?\]|[A-Z][A-Z0-9_]*),`).FindStringSubmatch(call) + if m == nil { + return nil, errors.New("no urls") + } + list := m[1] + if !strings.HasPrefix(list, "[") { + def := regexp.MustCompile(`(?ms)^` + list + ` = (\[.*?^\])`).FindStringSubmatch(module) + if def == nil { + return nil, errors.New("urls names " + list + ", which MODULE.bazel does not define as a list") + } + list = def[1] + } + var urls []string + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(list, -1) { + urls = append(urls, q[1]) + } + if len(urls) == 0 { + return nil, errors.New("no urls") + } + return urls, nil +} + // checkHermeticLLVMSlice: a sliced llvm_dist downloads only from a // gastownhall release, and its members equal the repack script's list, so // the pinned slice can be rebuilt from upstream and checked file by file. @@ -240,12 +305,16 @@ func TestBazelHermeticCCToolchainGuards(t *testing.T) { module := `bazel_dep(name = "rules_go", version = "0.63.0") bazel_dep(name = "toolchains_llvm", version = "1.11.0") +NOBLE_DEB_URLS = [ + "https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb", + "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/{path}", + "https://launchpad.net/ubuntu/+archive/primary/+files/{file}", + "https://archive.ubuntu.com/ubuntu/{path}", +] + deb_sysroot( name = "cc_sysroot_noble_amd64", - mirrors = [ - "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/", - "https://archive.ubuntu.com/ubuntu/", - ], + urls = NOBLE_DEB_URLS, packages = { "pool/main/g/glibc/libc6-dev_2.39-0ubuntu8.9_amd64.deb": "` + sum + `", }, @@ -293,7 +362,14 @@ llvm.sysroot( "other-host dist version": strings.Replace(module, "llvm_version = LLVM_VERSION", `llvm_version = "17.0.6"`, 1), "other-host dist versions": strings.Replace(module, `llvm_versions = {"": LLVM_VERSION},`, "", 1), "unpinned LLVM": strings.Replace(module, ` sha256 = "`+sum+`",`+"\n)", "\n)", 1), - "moving mirror first": strings.Replace(module, `"https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/",`, "", 1), + "moving mirror first": strings.Replace(module, `"https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb",`, `"https://archive.ubuntu.com/ubuntu/{path}",`, 1), + "no snapshot": strings.Replace(module, `"https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/{path}",`, "", 1), + "moving snapshot": strings.Replace(module, "/ubuntu/20261001T000000Z/{path}", "/ubuntu/{path}", 1), + "single host": strings.NewReplacer(`"https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb",`, "", `"https://launchpad.net/ubuntu/+archive/primary/+files/{file}",`, "", `"https://archive.ubuntu.com/ubuntu/{path}",`, "").Replace(module), + "package-blind url": strings.Replace(module, "/+files/{file}", "/+files/libc6.deb", 1), + "unknown placeholder": strings.Replace(module, "/+files/{file}", "/+files/{name}", 1), + "undefined url list": strings.Replace(module, "urls = NOBLE_DEB_URLS,", "urls = OTHER_URLS,", 1), + "no urls": strings.Replace(module, "urls = NOBLE_DEB_URLS,", "", 1), "unpinned package": strings.Replace(module, `_amd64.deb": "`+sum+`"`, `_amd64.deb": ""`, 1), "ICU in sysroot": strings.Replace(module, " packages = {\n", " packages = {\n \"pool/main/i/icu/libicu74_74.2-1ubuntu3.1_amd64.deb\": \""+sum+"\",\n", 1), "sysroot other targets": strings.Replace(module, "label = \""+hermeticCCSysrootLabel+"\",\n targets = [\"linux-x86_64\"]", "label = \""+hermeticCCSysrootLabel+"\",\n targets = []", 1), diff --git a/third_party/cc_toolchain/deb_sysroot.bzl b/third_party/cc_toolchain/deb_sysroot.bzl index ae96aa392..809476f93 100644 --- a/third_party/cc_toolchain/deb_sysroot.bzl +++ b/third_party/cc_toolchain/deb_sysroot.bzl @@ -1,7 +1,9 @@ """deb_sysroot: a C/C++ sysroot assembled from pinned Debian/Ubuntu packages. Every package is fetched by sha256, so the sysroot is byte-identical on every -host. It gives the hermetic LLVM toolchain (MODULE.bazel) the glibc, +host whichever source served it; the sources are only an availability chain. +All packages download concurrently (Bazel bounds it by +--http_max_parallel_downloads), each trying its URLs in order. It gives the hermetic LLVM toolchain (MODULE.bazel) the glibc, libstdc++ and ICU headers and link libraries that cgo targets need, instead of whatever the host has under /usr. """ @@ -23,20 +25,30 @@ grep -rlI --include='*.so' 'GROUP' usr/lib | while IFS= read -r s; do done """ +def _package_urls(templates, path, sha256): + return [ + t.format(path = path, file = path.rsplit("/", 1)[-1], sha256 = sha256) + for t in templates + ] + def _deb_sysroot_impl(rctx): + pending = [] for path, sha256 in rctx.attr.packages.items(): - name = path.rsplit("/", 1)[-1] - deb = "_debs/" + name - rctx.download( - url = [mirror + path for mirror in rctx.attr.mirrors], + deb = "_debs/" + path.rsplit("/", 1)[-1] + pending.append((deb, rctx.download( + url = _package_urls(rctx.attr.urls, path, sha256), output = deb, sha256 = sha256, - ) - unpacked = "_debs/" + name + ".d" + block = False, + ))) + for _, download in pending: + download.wait() + for deb, _ in pending: + unpacked = deb + ".d" rctx.extract(deb, output = unpacked) data = [f for f in rctx.path(unpacked).readdir() if f.basename.startswith("data.tar")] if len(data) != 1: - fail("%s: expected one data.tar.* member, found %s" % (name, data)) + fail("%s: expected one data.tar.* member, found %s" % (deb, data)) rctx.extract(data[0], output = ".") rctx.delete("_debs") @@ -67,11 +79,12 @@ deb_sysroot = repository_rule( attrs = { "packages": attr.string_dict( mandatory = True, - doc = "Pool path (relative to each mirror) to the .deb's sha256.", + doc = "Pool path (pool//...) to the .deb's sha256.", ), - "mirrors": attr.string_list( + "urls": attr.string_list( mandatory = True, - doc = "Archive roots tried in order; the first is an immutable snapshot.", + doc = "URL templates tried in order for every package: {path} is the pool " + + "path, {file} its basename and {sha256} its pinned digest.", ), "remove": attr.string_list( doc = "Bash globs (globstar) dropped after unpacking: docs, unused runtimes and archives.", From 7896abbfa8f6e791d20f4ae96a27ed9f3fa33077 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 18:22:08 +0000 Subject: [PATCH 018/151] ci(bazel): scope the proxied lane's shared-server projects to their own database (44 -> 34 shards) (#7436) * ci(bazel): scope the proxied lane's shared-server projects to their own database (44 -> 28 shards) //cmd/bd:bd_proxied_test spends nearly all its time in `bd init --proxied-server` against the shard's shared dolt sql-server. Measured where that goes (dolt query log): of ~2,031 statements per init, the ~220 INFORMATION_SCHEMA queries (migration guards and pre-repair probes) are ~80% of server time at a handful of databases, the 81 DOLT_COMMITs only ~0.5s. Dolt evaluates INFORMATION_SCHEMA by walking every database the session can see, so as root each init cost the server ~1.2 CPU-seconds more per database already on it (locally 6.7 CPU-s with 1 database, 21 with 11): a shard's k-th init paid for the k-1 projects before it. - proxied_shared_harness_test.go: newSharedProxiedProject now provisions the database server-side (root creates it plus a passwordless user of the same name with ALL on it and nothing else) and inits with --proxied-server-external-user, so each project's sessions see one database. newSharedProxiedRootProject keeps the old root/bd-creates-it shape for the four tests that need a server-wide privilege (DOLT_REMOTE remove, CREATE DATABASE, one compact statement); a run with the server log in test outputs showed no other denied statement. - Dropping finished databases instead was tried and rejected: a DROP DATABASE fails other sessions' in-flight INFORMATION_SCHEMA queries ("no root value found in session"), ~35 init failures per run. - No assertion changed. The bd-creates-the-database path stays covered by the root variant, the managed-local and non-shared init tests. - proxied_test_durations.json re-measured (two rbe-west runs): 19,618 -> 6,345 cost-seconds. 44 -> 28 shards, LPT-packed to ~227 per shard. rbe-west, --nocache_test_results (BEP attempt durations): before 44 shards, sum 6,063s, max 186s; scoped users at 44 shards, sum 2,337s, max 68s; at 20 shards, sum 1,950s, max 136s. Bead: ga-vnycm2.16 Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): proxied lane 28 -> 34 shards for a <=100s slowest shard PR CI at 28 shards (~227 packed cost each): 2,285 shard-seconds, median 78s, but max 113s; equally packed shards spread up to 1.45x the median. 34 shards (~187 each) targets a slowest shard under 100s. Bead: ga-vnycm2.16 Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/proxied-cmd-test-shards.txt | 430 ++++++++-------- .github/workflows/bazel.yml | 23 +- cmd/bd/BUILD.bazel | 6 +- ..._remote_remove_proxied_integration_test.go | 6 +- .../history_remote_frontdoor_matrix_test.go | 2 +- ...nance_commands_proxied_integration_test.go | 2 +- cmd/bd/proxied_shared_harness_test.go | 82 +++- ...multistatement_proxied_integration_test.go | 2 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 2 +- engdocs/TESTING.md | 4 +- scripts/ci/gen_proxied_shard_manifest.py | 7 +- scripts/ci/proxied_test_durations.json | 461 +++++++++--------- scripts/shard_manifest_test.go | 2 +- 13 files changed, 546 insertions(+), 483 deletions(-) diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 195ff2c5f..42a1fc999 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -225,12 +225,12 @@ # # Format: # -# 44-shard split for the Bazel-only proxied-server tier +# 34-shard split for the Bazel-only proxied-server tier # (bazel-proxied in .github/workflows/bazel.yml), bin-packed # longest-processing-time-first by measured wall-time from # scripts/ci/proxied_test_durations.json (see that file for # provenance and its "relative weight, not absolute SLA" caveat). -# Regenerate with scripts/ci/gen_proxied_shard_manifest.py 44 +# Regenerate with scripts/ci/gen_proxied_shard_manifest.py 34 # --weights=duration --write after adding or removing # TestProxiedServer*/TestServerMode* functions. --write is incremental: # it keeps every already-assigned test on its current shard and only @@ -252,254 +252,244 @@ # regeneration. Add them to the generator or the durations file # instead. -44 1 TestProxiedServerDep2 -44 1 TestProxiedServerServeAddDependencies +34 1 TestProxiedServerDep2 +34 1 TestProxiedServerMolProgress +34 1 TestProxiedServerReadyClaimConcurrent +34 1 TestProxiedServerServeActivatesTheEventsJournal +34 1 TestProxiedServerServeRelease -44 2 TestProxiedServerClose2 -44 2 TestProxiedServerCount -44 2 TestProxiedServerEventsJournalOffByDefault -44 2 TestProxiedServerServeRefusesAStreamOnADisabledJournal +34 2 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge +34 2 TestProxiedServerGateList +34 2 TestProxiedServerList +34 2 TestProxiedServerServeRemoveDependency +34 2 TestProxiedServerUpdateC -44 3 TestProxiedServerEventsJournal -44 3 TestProxiedServerServeReadsTheJournalItJustWrote -44 3 TestProxiedServerUpdate3 -44 3 TestServerModeSharedServeGlobalReportsTheServedDatabase +34 3 TestProxiedServerAutoPrune +34 3 TestProxiedServerClose2B +34 3 TestProxiedServerEventsJournalOffByDefault +34 3 TestProxiedServerMolLastActivity +34 3 TestProxiedServerPurgeWispsPlaneRetention +34 3 TestProxiedServerServeLifecycle +34 3 TestProxiedServerStatuses -44 4 TestProxiedServerClose2B -44 4 TestProxiedServerOutageReconnectAcceptanceMatrix -44 4 TestProxiedServerServeDelete -44 4 TestProxiedServerSetState +34 4 TestProxiedServerMolStale +34 4 TestProxiedServerReadyConcurrent +34 4 TestProxiedServerServeRelatedIssues +34 4 TestProxiedServerServeSettingsWrites +34 4 TestProxiedServerShow2 +34 4 TestServerModeServeSkipsPostRunMaintenance -44 5 TestProxiedServerCreateInfraTypeRoutesToWisps -44 5 TestProxiedServerHuman -44 5 TestProxiedServerListComments -44 5 TestProxiedServerServeClaimNext +34 5 TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix +34 5 TestProxiedServerInfo +34 5 TestProxiedServerNote +34 5 TestProxiedServerOutageReconnectAcceptanceMatrix +34 5 TestProxiedServerUpdate3 +34 5 TestServerModeDeferAutoWake -44 6 TestProxiedServerAssign -44 6 TestProxiedServerCompactDolt -44 6 TestProxiedServerHistory -44 6 TestProxiedServerIfRevisionGuardDelete -44 6 TestProxiedServerKV -44 6 TestProxiedServerServeStreamsTheJournalItIsWriting +34 6 TestProxiedServerPing +34 6 TestProxiedServerQuick +34 6 TestProxiedServerServeReadsTheJournalItJustWrote +34 6 TestProxiedServerSwarm +34 6 TestProxiedServerUpdate2B +34 6 TestProxiedServerWispList -44 7 TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce -44 7 TestProxiedServerDoltLifecycle -44 7 TestProxiedServerGateFrontDoorDirectServerProxyParity -44 7 TestProxiedServerIfRevisionGuardClose -44 7 TestProxiedServerMolDistill -44 7 TestProxiedServerPathHelpers -44 7 TestProxiedServerStorageClass -44 7 TestServerModeServe +34 7 TestProxiedServerCloseConcurrent +34 7 TestProxiedServerCreateInfraTypeRoutesToWisps +34 7 TestProxiedServerExternalCreate +34 7 TestProxiedServerIfRevisionGuardDelete +34 7 TestProxiedServerLabelC +34 7 TestProxiedServerTypesAgreeWithCreate -44 8 TestProxiedServerDelete -44 8 TestProxiedServerPourRejectsEnumViolatingVar -44 8 TestProxiedServerSQL -44 8 TestProxiedServerTag -44 8 TestProxiedServerUpdate2B +34 8 TestProxiedServerClose2 +34 8 TestProxiedServerDeleteWisp +34 8 TestProxiedServerExport +34 8 TestProxiedServerFindDuplicates +34 8 TestProxiedServerGateCheck +34 8 TestProxiedServerServeReadParity -44 9 TestProxiedServerConfigSetMany -44 9 TestProxiedServerReopen -44 9 TestProxiedServerServeClaim -44 9 TestProxiedServerUnclaimIfAssigneeParity -44 9 TestProxiedServerUpdateConcurrentAppendNotes +34 9 TestProxiedServerCloseSiblingBlockersFanIn +34 9 TestProxiedServerGateLifecycle +34 9 TestProxiedServerHistoryRemoteSupportedFrontDoorParity +34 9 TestProxiedServerIfRevisionGuardReopen +34 9 TestProxiedServerServeRefusesTheJournalWhenItIsDisabled +34 9 TestProxiedServerUpdate +34 9 TestServerModeServe -44 10 TestProxiedServerDeleteBlockedJSONSingleDoc -44 10 TestProxiedServerLink -44 10 TestProxiedServerPriority -44 10 TestProxiedServerReadyWisp -44 10 TestProxiedServerServeAddComment +34 10 TestProxiedServerContext +34 10 TestProxiedServerCreate2B +34 10 TestProxiedServerGateCheckB +34 10 TestProxiedServerServeRefusesAStreamOnADisabledJournal +34 10 TestProxiedServerTypes +34 10 TestProxiedServerUpdate2C -44 11 TestProxiedServerAutoPrune -44 11 TestProxiedServerBatchParity -44 11 TestProxiedServerContext -44 11 TestProxiedServerCreate2 -44 11 TestProxiedServerExternalMutationPolicy +34 11 TestProxiedServerBatchParity +34 11 TestProxiedServerCompactHistory +34 11 TestProxiedServerPour +34 11 TestProxiedServerQuery +34 11 TestProxiedServerReady +34 11 TestProxiedServerServeAddDependencies -44 12 TestProxiedServerExternalCreate -44 12 TestProxiedServerServeRelatedIssues -44 12 TestProxiedServerUpdate -44 12 TestProxiedServerWispList -44 12 TestServerModeTypesAgreeWithCreate +34 12 TestProxiedServerBlocked +34 12 TestProxiedServerChildren +34 12 TestProxiedServerCleanDatabases +34 12 TestProxiedServerReadyB +34 12 TestProxiedServerReadyWisp +34 12 TestProxiedServerReclaim +34 12 TestProxiedServerUnclaim -44 13 TestProxiedServerCreate -44 13 TestProxiedServerRecomputeBlocked -44 13 TestProxiedServerServeUpdate -44 13 TestProxiedServerShow2 -44 13 TestProxiedServerStale +34 13 TestProxiedServerCompactDolt +34 13 TestProxiedServerCreate2 +34 13 TestProxiedServerGraph +34 13 TestProxiedServerMolSquash +34 13 TestProxiedServerPromote +34 13 TestProxiedServerServeRefusesAStaleCheckpointWithGone -44 14 TestProxiedServerMolSquash -44 14 TestProxiedServerQuick -44 14 TestProxiedServerServeReadParity -44 14 TestProxiedServerServeRefusesAStaleCheckpointWithGone -44 14 TestProxiedServerShow2B +34 14 TestProxiedServerAssign +34 14 TestProxiedServerCloseB +34 14 TestProxiedServerDuplicates +34 14 TestProxiedServerHeartbeat +34 14 TestProxiedServerInitIfMissing +34 14 TestProxiedServerMolReadyGated -44 15 TestProxiedServerCountIncludeInfra -44 15 TestProxiedServerDeleteForceOrphans -44 15 TestProxiedServerDuplicates -44 15 TestProxiedServerLabel -44 15 TestProxiedServerServeBatchClose +34 15 TestProxiedServerDepConcurrent +34 15 TestProxiedServerImport +34 15 TestProxiedServerMoleculeFrontDoorWriteParity +34 15 TestProxiedServerReopenConcurrent +34 15 TestProxiedServerSQL -44 16 TestProxiedServerHistoryRemoteSupportedFrontDoorParity -44 16 TestProxiedServerQuery -44 16 TestProxiedServerSQLDatabaseFlag -44 16 TestProxiedServerShowB -44 16 TestProxiedServerWispCreate +34 16 TestProxiedServerBackupRefusedOnExternalTopology +34 16 TestProxiedServerCascadeParity +34 16 TestProxiedServerConfigSetMany +34 16 TestProxiedServerKV +34 16 TestProxiedServerServeCount +34 16 TestProxiedServerServeCreate +34 16 TestProxiedServerUpdateHooks -44 17 TestProxiedServerDoltRemoteRemove -44 17 TestProxiedServerImport -44 17 TestProxiedServerLint -44 17 TestProxiedServerServeEdgeCounts -44 17 TestProxiedServerStatus +34 17 TestProxiedServerDep +34 17 TestProxiedServerMultiStatementSQL +34 17 TestProxiedServerOrphans +34 17 TestProxiedServerPrime +34 17 TestProxiedServerSQLDatabaseFlag +34 17 TestProxiedServerServeModeGate -44 18 TestProxiedServerCloseConcurrent -44 18 TestProxiedServerDep2C -44 18 TestProxiedServerEdit -44 18 TestProxiedServerIfRevisionGuardUpdate -44 18 TestProxiedServerSearch +34 18 TestProxiedServerClose3 +34 18 TestProxiedServerCount +34 18 TestProxiedServerCountIncludeInfra +34 18 TestProxiedServerDoltRemoteRemove +34 18 TestProxiedServerMolShow +34 18 TestProxiedServerWispCreateRejectsEnumViolatingVar -44 19 TestProxiedServerServeClose -44 19 TestProxiedServerTypesAgreeWithCreate -44 19 TestProxiedServerUpdate2 -44 19 TestProxiedServerUpdateClosePolicy +34 19 TestProxiedServerCreate +34 19 TestProxiedServerMolCurrent +34 19 TestProxiedServerServeBatchClose +34 19 TestProxiedServerShow3 +34 19 TestServerModeTypesAgreeWithCreate -44 20 TestProxiedServerDeferAutoWake -44 20 TestProxiedServerEpic -44 20 TestProxiedServerIfRevisionGuardReopen -44 20 TestProxiedServerInitIfMissing -44 20 TestProxiedServerReopenB -44 20 TestProxiedServerUpdateIfGuards +34 20 TestProxiedServerDelete +34 20 TestProxiedServerLint +34 20 TestProxiedServerPriority +34 20 TestProxiedServerShow2B +34 20 TestProxiedServerWispGC +34 20 TestServerModeSharedServeGlobalReportsTheServedDatabase -44 21 TestProxiedServerBlocked -44 21 TestProxiedServerCleanDatabases -44 21 TestProxiedServerConfig -44 21 TestProxiedServerMolShow -44 21 TestProxiedServerPing -44 21 TestProxiedServerServeActivatesTheEventsJournal +34 21 TestProxiedServerConfig +34 21 TestProxiedServerExternalPolicyDirectParity +34 21 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity +34 21 TestProxiedServerPathHelpers +34 21 TestProxiedServerPrune +34 21 TestProxiedServerServeClose +34 21 TestProxiedServerUnclaimIfAssigneeParity -44 22 TestProxiedServerCloseBeadGate -44 22 TestProxiedServerExternalPolicyDirectParity -44 22 TestProxiedServerFindDuplicates -44 22 TestProxiedServerReadyB -44 22 TestProxiedServerServeCreate +34 22 TestProxiedServerGateCreateInheritsRepoMetadata +34 22 TestProxiedServerRecomputeBlocked +34 22 TestProxiedServerServeEdgeCounts +34 22 TestProxiedServerTag +34 22 TestProxiedServerUpdateB +34 22 TestProxiedServerUpdateConcurrentMetadata -44 23 TestProxiedServerCloseSiblingBlockersFanIn -44 23 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity -44 23 TestProxiedServerPour -44 23 TestProxiedServerShow3 -44 23 TestProxiedServerUpdateConcurrentClaim +34 23 TestProxiedServerDep2B +34 23 TestProxiedServerIfRevisionGuardUpdate +34 23 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows +34 23 TestProxiedServerMolBurn +34 23 TestProxiedServerServeUpdate +34 23 TestProxiedServerUpdateConcurrentClaim +34 23 TestProxiedServerUpdateWisp -44 24 TestProxiedServerLabelC -44 24 TestProxiedServerMemory -44 24 TestProxiedServerMolStale -44 24 TestProxiedServerServeUpdateClaim -44 24 TestProxiedServerSwarm +34 24 TestProxiedServerDeleteBlockedJSONSingleDoc +34 24 TestProxiedServerDeleteForceOrphans +34 24 TestProxiedServerHuman +34 24 TestProxiedServerIfRevisionGuardAssign +34 24 TestProxiedServerListComments +34 24 TestProxiedServerListSortRetiresTheWalk +34 24 TestProxiedServerShowWatch +34 24 TestProxiedServerUpdateIfGuards -44 25 TestProxiedServerHeartbeat -44 25 TestProxiedServerMolLastActivity -44 25 TestProxiedServerReassignFence -44 25 TestProxiedServerStatuses -44 25 TestProxiedServerUpdateWisp +34 25 TestProxiedServerInitExplicitBeadsDirIsAuthoritative +34 25 TestProxiedServerSetState +34 25 TestProxiedServerShow +34 25 TestProxiedServerShowB +34 25 TestProxiedServerStale +34 25 TestProxiedServerWispCreate -44 26 TestProxiedServerGraph -44 26 TestProxiedServerMolBond -44 26 TestProxiedServerServeReadRevision -44 26 TestProxiedServerTodo -44 26 TestProxiedServerUpdateHooks +34 26 TestProxiedServerDeferAutoWake +34 26 TestProxiedServerExternalMutationPolicy +34 26 TestProxiedServerReopenB +34 26 TestProxiedServerSearch +34 26 TestProxiedServerServeClaim +34 26 TestProxiedServerShow3B -44 27 TestProxiedServerCreate2B -44 27 TestProxiedServerGateList -44 27 TestProxiedServerGatedRendering -44 27 TestProxiedServerReopenConcurrent -44 27 TestProxiedServerSQLStatementRouting +34 27 TestProxiedServerDeferUndefer +34 27 TestProxiedServerGatedRendering +34 27 TestProxiedServerHistory +34 27 TestProxiedServerLabel +34 27 TestProxiedServerLabelB +34 27 TestProxiedServerUpdateClosePolicy -44 28 TestProxiedServerComment -44 28 TestProxiedServerMolProgress -44 28 TestProxiedServerServeCount -44 28 TestProxiedServerServeRemoveDependency -44 28 TestProxiedServerUpdateB +34 28 TestProxiedServerDep2C +34 28 TestProxiedServerDoltLifecycle +34 28 TestProxiedServerReady2 +34 28 TestProxiedServerReassignFence +34 28 TestProxiedServerSQLStatementRouting +34 28 TestProxiedServerServeClaimNext +34 28 TestProxiedServerUpdateConcurrentAppendNotes -44 29 TestProxiedServerCreate2C -44 29 TestProxiedServerDep -44 29 TestProxiedServerDepConcurrent -44 29 TestProxiedServerServeSettingsWrites -44 29 TestProxiedServerWispCreateRejectsEnumViolatingVar +34 29 TestProxiedServerGateFrontDoorDirectServerProxyParity +34 29 TestProxiedServerMolDistill +34 29 TestProxiedServerReopen +34 29 TestProxiedServerServeDelete +34 29 TestProxiedServerState -44 30 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge -44 30 TestProxiedServerInfo -44 30 TestProxiedServerPrime -44 30 TestProxiedServerUpdateC -44 30 TestServerModeServeSkipsPostRunMaintenance +34 30 TestProxiedServerDeleteConcurrent +34 30 TestProxiedServerReadyC +34 30 TestProxiedServerServeAddComment +34 30 TestProxiedServerStatus +34 30 TestProxiedServerTodo +34 30 TestProxiedServerUpdate2 -44 31 TestProxiedServerCleanDatabasesPurgeDropped -44 31 TestProxiedServerGateLifecycle -44 31 TestProxiedServerPurgeWispsPlaneRetention -44 31 TestProxiedServerShow3B -44 31 TestServerModeDeferAutoWake +34 31 TestProxiedServerCloseBeadGate +34 31 TestProxiedServerComment +34 31 TestProxiedServerConfigB +34 31 TestProxiedServerEventsJournal +34 31 TestProxiedServerIfRevisionGuardClose +34 31 TestProxiedServerMolBond -44 32 TestProxiedServerClose3 -44 32 TestProxiedServerGC -44 32 TestProxiedServerList -44 32 TestProxiedServerReadyConcurrent -44 32 TestProxiedServerServeModeGate +34 32 TestProxiedServerCleanDatabasesPurgeDropped +34 32 TestProxiedServerConfigConcurrent +34 32 TestProxiedServerDepB +34 32 TestProxiedServerServeStreamsTheJournalItIsWriting +34 32 TestProxiedServerServeUpdateClaim +34 32 TestProxiedServerStorageClass -44 33 TestProxiedServerDeferUndefer -44 33 TestProxiedServerDepB -44 33 TestProxiedServerInitExplicitBeadsDirIsAuthoritative -44 33 TestProxiedServerMolBurn -44 33 TestProxiedServerOrphans +34 33 TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce +34 33 TestProxiedServerClose +34 33 TestProxiedServerGC +34 33 TestProxiedServerMemory +34 33 TestProxiedServerPourRejectsEnumViolatingVar +34 33 TestProxiedServerRelate -44 34 TestProxiedServerChildren -44 34 TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix -44 34 TestProxiedServerPromote -44 34 TestProxiedServerServeRelease -44 34 TestProxiedServerUpdate2C - -44 35 TestProxiedServerConfigConcurrent -44 35 TestProxiedServerReady2 -44 35 TestProxiedServerShow - -44 36 TestProxiedServerDep2B -44 36 TestProxiedServerGateCreateInheritsRepoMetadata -44 36 TestProxiedServerServeRefusesTheJournalWhenItIsDisabled -44 36 TestProxiedServerWispGC - -44 37 TestProxiedServerDeleteWisp -44 37 TestProxiedServerNote -44 37 TestProxiedServerReady -44 37 TestProxiedServerShowWatch - -44 38 TestProxiedServerBackupRefusedOnExternalTopology -44 38 TestProxiedServerCascadeParity -44 38 TestProxiedServerClose4 -44 38 TestProxiedServerReadyClaimConcurrent -44 38 TestProxiedServerReclaim - -44 39 TestProxiedServerGateCheck -44 39 TestProxiedServerLabelB -44 39 TestProxiedServerMoleculeFrontDoorWriteParity - -44 40 TestProxiedServerDeleteConcurrent -44 40 TestProxiedServerIfRevisionGuardAssign -44 40 TestProxiedServerReadyC -44 40 TestProxiedServerRelate - -44 41 TestProxiedServerMolCurrent -44 41 TestProxiedServerPrune -44 41 TestProxiedServerTypes -44 41 TestProxiedServerUnclaim - -44 42 TestProxiedServerClose -44 42 TestProxiedServerCompactHistory -44 42 TestProxiedServerState -44 42 TestProxiedServerUpdateConcurrentMetadata - -44 43 TestProxiedServerCloseB -44 43 TestProxiedServerConfigB -44 43 TestProxiedServerMultiStatementSQL - -44 44 TestProxiedServerExport -44 44 TestProxiedServerGateCheckB -44 44 TestProxiedServerListSortRetiresTheWalk -44 44 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows -44 44 TestProxiedServerMolReadyGated -44 44 TestProxiedServerServeLifecycle +34 34 TestProxiedServerClose4 +34 34 TestProxiedServerCreate2C +34 34 TestProxiedServerEdit +34 34 TestProxiedServerEpic +34 34 TestProxiedServerLink +34 34 TestProxiedServerServeReadRevision diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5922cf041..1be23c9fa 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1732,8 +1732,8 @@ jobs: # Like pr-risk.yml's proxied-server tier, "Test (Proxied Dolt Cmd N/15)", # as --config=doltserver-proxied: //cmd/bd:bd_proxied_test runs the shard # script, so shard k runs exactly that manifest entry's tests. This lane - # runs 44 shards of its own (.github/scripts/proxied-cmd-test-shards.txt's - # duration-balanced 44-block; see scripts/ci/proxied_test_durations.json), + # runs 34 shards of its own (.github/scripts/proxied-cmd-test-shards.txt's + # duration-balanced 34-block; see scripts/ci/proxied_test_durations.json), # not the legacy jobs' 15 — shard k here is not the same split as their # shard k. Same build as --config=doltserver (race bd_test, non-race # bd_for_tests), shared with bazel-doltserver through the remote cache. @@ -1793,15 +1793,16 @@ jobs: fi # Remote: each shard's Go timeout is its job's 15m, 1200s Bazel cap, - # plus a compile. The 44-shard block is LPT-packed by + # plus a compile. The 34-shard block is LPT-packed by # scripts/ci/proxied_test_durations.json, a rbe-west capture of each - # top-level test's summed leaf-test time (see its header). Packed to - # <=430 cost-seconds per shard, and a shard's wall time ran ~0.3x its - # packed cost on the lane's executors (-test.parallel=4), so shards - # are expected around 100-150s. Nearly all of that is `bd init - # --proxied-server` against the shard's dolt sql-server, ~one per - # subtest; a shard running far longer than its packed cost is worth - # comparing against this job's recent history before assuming a hang. + # top-level test's summed leaf-test time (see its header), to ~187 + # cost-seconds per shard; shards are expected around 55-95s (the same + # packed cost has run up to 1.45x the median wall time). Nearly + # all of that is `bd init --proxied-server` against the shard's dolt + # sql-server, ~one per subtest, each as a user scoped to its own + # database (cmd/bd/proxied_shared_harness_test.go); a shard running far + # longer is worth comparing against this job's recent history before + # assuming a hang. - name: bazel test //... --config=doltserver-proxied id: test timeout-minutes: 25 @@ -1830,7 +1831,7 @@ jobs: run: >- python3 tools/bazel/check_shard_coverage.py --bep "$RUNNER_TEMP/bazel-bep.json" - --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 44 + --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 34 - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 6dc8c19db..f725268ee 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1443,8 +1443,8 @@ sh_test( tags = ["embedded"], ) -# Runs 44 Bazel shards against the TestProxiedServer*/TestServerMode* -# functions, assigned by the manifest's 44-shard block (duration-balanced; +# Runs 34 Bazel shards against the TestProxiedServer*/TestServerMode* +# functions, assigned by the manifest's 34-shard block (duration-balanced; # see scripts/ci/proxied_test_durations.json), against this configuration's # (race) bd_test with BEADS_TEST_PROXIED_SERVER=1 and the non-race # bd_for_tests as the subprocess binary. This is a *different* shard count @@ -1501,7 +1501,7 @@ sh_test( # 192-core worker). "GOMAXPROCS": "4", }, - shard_count = 44, + shard_count = 34, tags = ["dolt-server-proxied"], ) diff --git a/cmd/bd/dolt_remote_remove_proxied_integration_test.go b/cmd/bd/dolt_remote_remove_proxied_integration_test.go index 77fb06592..ca589fbb4 100644 --- a/cmd/bd/dolt_remote_remove_proxied_integration_test.go +++ b/cmd/bd/dolt_remote_remove_proxied_integration_test.go @@ -36,7 +36,7 @@ func TestProxiedServerDoltRemoteRemove(t *testing.T) { t.Run("removes_existing_remote", func(t *testing.T) { t.Parallel() - p := newSharedProxiedProject(t, bd, "prr1") + p := newSharedProxiedRootProject(t, bd, "prr1") seedRemote(t, p, "backup", "https://doltremoteapi.dolthub.com/org/backup") out, err := bdProxiedRun(t, bd, p.dir, "dolt", "remote", "remove", "backup") @@ -53,7 +53,7 @@ func TestProxiedServerDoltRemoteRemove(t *testing.T) { t.Run("json_output", func(t *testing.T) { t.Parallel() - p := newSharedProxiedProject(t, bd, "prr2") + p := newSharedProxiedRootProject(t, bd, "prr2") seedRemote(t, p, "backup", "https://doltremoteapi.dolthub.com/org/backup") out, err := bdProxiedRun(t, bd, p.dir, "dolt", "remote", "remove", "--json", "backup") @@ -82,7 +82,7 @@ func TestProxiedServerDoltRemoteRemove(t *testing.T) { t.Run("nonexistent_remote_errors", func(t *testing.T) { t.Parallel() - p := newSharedProxiedProject(t, bd, "prr3") + p := newSharedProxiedRootProject(t, bd, "prr3") out, err := bdProxiedRun(t, bd, p.dir, "dolt", "remote", "remove", "ghost") if err == nil { t.Fatalf("expected error removing nonexistent remote, got:\n%s", out) diff --git a/cmd/bd/history_remote_frontdoor_matrix_test.go b/cmd/bd/history_remote_frontdoor_matrix_test.go index 8941651af..2a6f33ea2 100644 --- a/cmd/bd/history_remote_frontdoor_matrix_test.go +++ b/cmd/bd/history_remote_frontdoor_matrix_test.go @@ -357,7 +357,7 @@ func TestProxiedServerHistoryRemoteSupportedFrontDoorParity(t *testing.T) { }}, {"external-tcp", func(t *testing.T) proxiedProject { requireProxiedServerEnv(t) - return newSharedProxiedProject(t, bd, "hs_tcp") + return newSharedProxiedRootProject(t, bd, "hs_tcp") }}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/bd/maintenance_commands_proxied_integration_test.go b/cmd/bd/maintenance_commands_proxied_integration_test.go index 68fa7d3bf..549d8d2a3 100644 --- a/cmd/bd/maintenance_commands_proxied_integration_test.go +++ b/cmd/bd/maintenance_commands_proxied_integration_test.go @@ -183,7 +183,7 @@ func TestProxiedServerCompactHistory(t *testing.T) { requireSharedProxiedServer(t) t.Parallel() bd := buildEmbeddedBD(t) - p := newSharedProxiedProject(t, bd, "cphist") + p := newSharedProxiedRootProject(t, bd, "cphist") for _, title := range []string{"keep-alpha", "keep-beta", "keep-gamma"} { bdProxiedCreate(t, bd, p.dir, title, "--type", "task") diff --git a/cmd/bd/proxied_shared_harness_test.go b/cmd/bd/proxied_shared_harness_test.go index cbaf3c5dd..2d34fbfee 100644 --- a/cmd/bd/proxied_shared_harness_test.go +++ b/cmd/bd/proxied_shared_harness_test.go @@ -3,13 +3,17 @@ package main import ( + "context" + "database/sql" "fmt" "os" "strconv" "sync" "sync/atomic" "testing" + "time" + "github.com/steveyegge/beads/internal/storage/doltutil" "github.com/steveyegge/beads/internal/testutil" ) @@ -55,22 +59,88 @@ func uniqueProxiedDatabase() string { return fmt.Sprintf("bdtest_%d", sharedProxiedDBSeq.Add(1)) } -func sharedProxiedInitArgs(t *testing.T, extraInitArgs ...string) (string, []string) { +// sharedProxiedCredential is who a shared-server project's `bd init` (and so +// its proxy) connects to the shard's dolt sql-server as. +type sharedProxiedCredential int + +const ( + // sharedProxiedScopedUser: the default. The database is provisioned + // server-side and the project connects as a user that can see only it + // (see provisionSharedProxiedDatabase). + sharedProxiedScopedUser sharedProxiedCredential = iota + // sharedProxiedRoot: the project connects as root and `bd init` creates + // the database itself. Only for tests that need a server-wide privilege + // (CALL DOLT_REMOTE, creating another database); each such init pays for + // every database on the server. + sharedProxiedRoot +) + +func sharedProxiedInitArgs(t *testing.T, cred sharedProxiedCredential, extraInitArgs ...string) (string, []string) { t.Helper() port := requireSharedProxiedServer(t) database := uniqueProxiedDatabase() - args := append([]string{ + args := []string{ "--database", database, "--proxied-server-external-host", "127.0.0.1", "--proxied-server-external-port", strconv.Itoa(port), - }, extraInitArgs...) - return database, args + } + if cred == sharedProxiedScopedUser { + provisionSharedProxiedDatabase(t, port, database) + args = append(args, "--proxied-server-external-user", database) + } + return database, append(args, extraInitArgs...) +} + +// provisionSharedProxiedDatabase provisions database on the shard's shared +// dolt sql-server the way a server that provisions databases server-side +// does: root creates the empty database and a passwordless user of the same +// name with ALL on it and nothing else, and the project connects as that user +// (--proxied-server-external-user). +// +// The scoping is what keeps a shard's inits from slowing each other down. +// Dolt answers an INFORMATION_SCHEMA query by walking every database the +// session can see, and `bd init`'s migration chain issues ~220 of them, so as +// root each init cost the server ~1.2 CPU-seconds more per database already on +// it: a shard's k-th init paid for the k-1 projects before it. A user that can +// see one database pays for one. Dropping finished projects' databases instead +// is not an option: a DROP DATABASE fails other sessions' in-flight +// INFORMATION_SCHEMA queries ("no root value found in session"). +func provisionSharedProxiedDatabase(t *testing.T, port int, database string) { + t.Helper() + dsn := doltutil.ServerDSN{Host: "127.0.0.1", Port: port, User: "root"}.String() + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("provisioning %s: opening the shared server: %v", database, err) + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + // database is uniqueProxiedDatabase's bdtest_: safe to splice. + for _, stmt := range []string{ + fmt.Sprintf("CREATE DATABASE `%s`", database), + fmt.Sprintf("CREATE USER '%s'@'%%'", database), + fmt.Sprintf("GRANT ALL ON `%s`.* TO '%s'@'%%'", database, database), + } { + if _, err := db.ExecContext(ctx, stmt); err != nil { + t.Fatalf("provisioning %s: %s: %v", database, stmt, err) + } + } } func newSharedProxiedProject(t *testing.T, bd, prefix string, extraInitArgs ...string) proxiedProject { t.Helper() - database, args := sharedProxiedInitArgs(t, extraInitArgs...) + database, args := sharedProxiedInitArgs(t, sharedProxiedScopedUser, extraInitArgs...) + p := bdProxiedInit(t, bd, prefix, args...) + p.database = database + return p +} + +// newSharedProxiedRootProject is newSharedProxiedProject connecting as root +// (sharedProxiedRoot), for a test that needs a server-wide privilege. +func newSharedProxiedRootProject(t *testing.T, bd, prefix string, extraInitArgs ...string) proxiedProject { + t.Helper() + database, args := sharedProxiedInitArgs(t, sharedProxiedRoot, extraInitArgs...) p := bdProxiedInit(t, bd, prefix, args...) p.database = database return p @@ -78,7 +148,7 @@ func newSharedProxiedProject(t *testing.T, bd, prefix string, extraInitArgs ...s func newSharedProxiedProjectWithHooks(t *testing.T, bd, prefix string, hooks map[string]string, extraInitArgs ...string) proxiedProject { t.Helper() - database, args := sharedProxiedInitArgs(t, extraInitArgs...) + database, args := sharedProxiedInitArgs(t, sharedProxiedScopedUser, extraInitArgs...) p := bdProxiedInitWithHooks(t, bd, prefix, hooks, args...) p.database = database return p diff --git a/cmd/bd/sql_multistatement_proxied_integration_test.go b/cmd/bd/sql_multistatement_proxied_integration_test.go index 0f5736a3b..54eccfe63 100644 --- a/cmd/bd/sql_multistatement_proxied_integration_test.go +++ b/cmd/bd/sql_multistatement_proxied_integration_test.go @@ -79,7 +79,7 @@ func TestProxiedServerMultiStatementSQL(t *testing.T) { func TestProxiedServerSQLDatabaseFlag(t *testing.T) { bd := buildEmbeddedBD(t) - p := newSharedProxiedProject(t, bd, "df") + p := newSharedProxiedRootProject(t, bd, "df") // Stand up a second database with a table, alongside the project database. // The server is shared, so the name derives from the project's unique one. diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 93ec7f4cd..68b9aeee6 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1041,7 +1041,7 @@ Required` requires them to have run remotely and passed. frozen 20-shard block was the retired `test-embedded-cmd` jobs' split, F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; the frozen 5-shard block was `test-embedded-storage`'s, F1), - `//cmd/bd:bd_proxied_test` (44; the frozen 15-shard block was + `//cmd/bd:bd_proxied_test` (34; the frozen 15-shard block was `test-proxied-cmd`'s, F2) and `//internal/storage/dolt:dolt_server_full_test` (16) to have run exactly the tests its shard script lists (list-only mode, minus diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 2f293a23b..65f838f42 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -42,7 +42,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | | Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | -| Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 44 shards, each with a Dolt server. | +| Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 34 shards, each with a Dolt server. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | @@ -184,7 +184,7 @@ builds with the integration tag like `--config=integration`. Each shard runs its CI job's shard script, so for `--config=doltserver-integration` Bazel shard k runs the tests of job k+1 (both split the manifest's 16-shard block the same way). `--config=doltserver-proxied`'s `bd_proxied_test` instead -runs the manifest's own 44-shard block — bin-packed by measured duration, +runs the manifest's own 34-shard block — bin-packed by measured duration, not the legacy jobs' 15-shard, bd-init-cost-proxy block — so shard k there is not job k+1's tests; it is a different split of the same tests. diff --git a/scripts/ci/gen_proxied_shard_manifest.py b/scripts/ci/gen_proxied_shard_manifest.py index 0a099e0f9..ecac80c89 100644 --- a/scripts/ci/gen_proxied_shard_manifest.py +++ b/scripts/ci/gen_proxied_shard_manifest.py @@ -56,7 +56,7 @@ name coverage. The manifest file holds more than one block (one per total_shards value in use): the legacy, frozen 15-shard block pr-risk.yml/main.yml's jobs read, and -the Bazel-only lane's own block (44 shards since 2026-10-09). By default this prints +the Bazel-only lane's own block (34 shards since 2026-10-09). By default this prints only the requested total's block to stdout, which is NOT safe to redirect straight into the manifest file (`gen... 30 > file` deletes every other block). Use --write to update the file in place instead: it replaces only @@ -81,6 +81,9 @@ import sys default_manifest_path = '.github/scripts/proxied-cmd-test-shards.txt' func_re = re.compile(r'^func (Test(?:ProxiedServer|ServerMode)[A-Za-z0-9_]+)\(') +# A shared-server bd init: newSharedProxiedProject(WithHooks) or the root +# variant, newSharedProxiedRootProject. +shared_init_re = re.compile(r'newSharedProxied(?:Root)?Project') def discover_inits_cost(): @@ -98,7 +101,7 @@ def discover_inits_cost(): elif ln.startswith('func '): cur = None if cur: - costs[cur] += ln.count('newSharedProxiedProject') + costs[cur] += len(shared_init_re.findall(ln)) for k in costs: costs[k] = max(costs[k], 1) return costs diff --git a/scripts/ci/proxied_test_durations.json b/scripts/ci/proxied_test_durations.json index 27297aae8..fa22729c8 100644 --- a/scripts/ci/proxied_test_durations.json +++ b/scripts/ci/proxied_test_durations.json @@ -3,35 +3,34 @@ "Measured on rbe-west, the lane's own remote executors (instance oss), on", "2026-10-09: two `bazel test //cmd/bd:bd_proxied_test", "--config=doltserver-proxied --nocache_test_results` runs, one against the", - "then-committed 30-shard block and one against a 40-shard repack, so every", - "discovered test ran exactly once per run, with the lane's -test.parallel=4", - "and GOMAXPROCS=4. Both passed.", + "then-committed 44-shard block and one against a 20-shard repack, both", + "with the shared harness's database-scoped users", + "(cmd/bd/proxied_shared_harness_test.go), so every discovered test ran", + "exactly once per run, with the lane's -test.parallel=4 and GOMAXPROCS=4.", + "Both passed.", "", "Each value is a top-level test's serial-equivalent cost: the sum of the", " of every leaf test under it (a subtest with no subtests of", "its own; the test itself when it has none), each leaf averaged over the", - "two runs. It is NOT , which the previous (local) capture", - "used: for a t.Parallel test that attribute runs from the test's first RUN", - "line to its last PASS line, so it also counts the time the test sat paused", - "behind its shard's other tests. That made a cheap test that happened to", - "wait look as heavy as the shard, which is why the previous 30-shard block,", - "packed to ~360s per shard on paper, ran 38-286s per shard in CI. Leaves,", - "not direct subtests, because a parallel group's own is 0.", + "two runs. It is NOT : for a t.Parallel test that", + "attribute runs from the test's first RUN line to its last PASS line, so it", + "also counts the time the test sat paused behind its shard's other tests.", + "Leaves, not direct subtests, because a parallel group's own is 0.", "", - "A shard's remote wall time came out at about 0.3x its packed cost (four", - "subtests run at once): correlation 0.90 over the 30-shard run (19,224", - "cost-seconds, 5,671 shard-seconds of wall time); the 40-shard run's", - "shards ran 71-162s, apart from one that held two then-unsplit tests whose", - "cost the first capture had missed.", + "Nearly all of a test's cost is `bd init --proxied-server` against the", + "shard's dolt sql-server, ~one per subtest. Before the scoped users each", + "init also paid for every database already on the server (Dolt walks every", + "database the session can see for each of the ~220 INFORMATION_SCHEMA", + "queries in the migration chain), so a test's cost depended on how many", + "projects its shard had created before it; the previous capture totalled", + "19,618 cost-seconds over the same tests, this one 6,345.", "", - "The cost is almost all `bd init --proxied-server` against the shared dolt", - "sql-server: each init issues ~2,000 statements (81 DOLT_COMMITs) and costs", - "the server ~6-12 CPU-seconds, and most subtests run one init. A top-level", - "test with ~20 single-init subtests therefore cost 600-920 and ran for up", - "to ~350s alone, which no shard count can pack under that. The fifteen", - "heaviest were split into two or three top-level functions", - "(TestProxiedServerUpdate, -UpdateB, -UpdateC, ...); a split function's", - "entry is the sum of the leaves it now holds.", + "A shard's remote wall time came out at about 0.33x its packed cost (four", + "subtests run at once), with a wide spread between equally packed shards:", + "a 20-shard repack (~335 per shard) ran median 113s, max 136s; a 28-shard", + "repack (~227) ran median 78s, max 113s in PR CI. Size the shard count for", + "the slowest shard, not the median.", "", "`seconds_per_init_fallback` is the sum of all costs over the sum of all", "inits (discover_inits_cost()), used only for a test with no entry yet.", @@ -40,215 +39,215 @@ "top-level test's leaf s from every shard's", "bazel-testlogs/cmd/bd/bd_proxied_test/shard_*/test.xml." ], - "seconds_per_init_fallback": 29.5374, + "seconds_per_init_fallback": 9.9294, "tests": { - "TestProxiedServerAssign": 55.49, - "TestProxiedServerAutoPrune": 6.71, - "TestProxiedServerBackupRefusedOnExternalTopology": 0.45, - "TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce": 10.13, - "TestProxiedServerBatchParity": 64.1, - "TestProxiedServerBlocked": 49.25, - "TestProxiedServerCascadeParity": 167.07, - "TestProxiedServerChildren": 12.43, - "TestProxiedServerCleanDatabases": 1.02, - "TestProxiedServerCleanDatabasesPurgeDropped": 2.23, - "TestProxiedServerClose": 223.82, - "TestProxiedServerClose2": 379.25, - "TestProxiedServerClose2B": 372.47, - "TestProxiedServerClose3": 253.62, - "TestProxiedServerClose4": 236.01, - "TestProxiedServerCloseB": 199.51, - "TestProxiedServerCloseBeadGate": 85.6, - "TestProxiedServerCloseConcurrent": 39.05, - "TestProxiedServerCloseSiblingBlockersFanIn": 45.72, - "TestProxiedServerComment": 101.7, - "TestProxiedServerCompactDolt": 0.15, - "TestProxiedServerCompactHistory": 2.27, - "TestProxiedServerConfig": 281.3, - "TestProxiedServerConfigB": 221.92, - "TestProxiedServerConfigConcurrent": 21.07, - "TestProxiedServerConfigSetMany": 56.32, - "TestProxiedServerContext": 30.68, - "TestProxiedServerCount": 29.54, - "TestProxiedServerCountIncludeInfra": 7.0, - "TestProxiedServerCreate": 35.39, - "TestProxiedServerCreate2": 313.1, - "TestProxiedServerCreate2B": 266.11, - "TestProxiedServerCreate2C": 115.17, - "TestProxiedServerCreateInfraTypeRoutesToWisps": 54.19, - "TestProxiedServerDeferAutoWake": 10.55, - "TestProxiedServerDeferUndefer": 23.62, - "TestProxiedServerDelete": 30.3, - "TestProxiedServerDeleteBlockedJSONSingleDoc": 16.89, - "TestProxiedServerDeleteConcurrent": 19.41, - "TestProxiedServerDeleteForceOrphans": 34.51, - "TestProxiedServerDeleteWisp": 4.98, - "TestProxiedServerDep": 265.67, - "TestProxiedServerDep2": 421.17, - "TestProxiedServerDep2B": 245.92, - "TestProxiedServerDep2C": 287.95, - "TestProxiedServerDepB": 252.49, - "TestProxiedServerDepConcurrent": 29.34, - "TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge": 23.65, - "TestProxiedServerDoltLifecycle": 0.19, - "TestProxiedServerDoltRemoteRemove": 79.1, - "TestProxiedServerDuplicates": 76.36, - "TestProxiedServerEdit": 19.49, - "TestProxiedServerEpic": 47.73, - "TestProxiedServerEventsJournal": 6.01, - "TestProxiedServerEventsJournalOffByDefault": 5.45, - "TestProxiedServerExport": 18.56, - "TestProxiedServerExternalCreate": 5.6, - "TestProxiedServerExternalMutationPolicy": 15.57, - "TestProxiedServerExternalPolicyDirectParity": 8.37, - "TestProxiedServerFindDuplicates": 51.33, - "TestProxiedServerGC": 5.82, - "TestProxiedServerGateCheck": 171.29, - "TestProxiedServerGateCheckB": 205.0, - "TestProxiedServerGateCreateInheritsRepoMetadata": 4.94, - "TestProxiedServerGateFrontDoorDirectServerProxyParity": 22.23, - "TestProxiedServerGateLifecycle": 124.1, - "TestProxiedServerGateList": 48.97, - "TestProxiedServerGatedRendering": 9.09, - "TestProxiedServerGraph": 8.3, - "TestProxiedServerHeartbeat": 11.7, - "TestProxiedServerHistory": 7.88, - "TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix": 26.2, - "TestProxiedServerHistoryRemoteSupportedFrontDoorParity": 5.58, - "TestProxiedServerHuman": 358.37, - "TestProxiedServerIfRevisionGuardAssign": 0.66, - "TestProxiedServerIfRevisionGuardClose": 0.82, - "TestProxiedServerIfRevisionGuardDelete": 0.81, - "TestProxiedServerIfRevisionGuardReopen": 1.48, - "TestProxiedServerIfRevisionGuardUpdate": 0.65, - "TestProxiedServerImport": 288.64, - "TestProxiedServerInfo": 6.0, - "TestProxiedServerInitExplicitBeadsDirIsAuthoritative": 12.59, - "TestProxiedServerInitIfMissing": 0.45, - "TestProxiedServerKV": 340.45, - "TestProxiedServerLabel": 298.09, - "TestProxiedServerLabelB": 234.22, - "TestProxiedServerLabelC": 271.95, - "TestProxiedServerLink": 62.35, - "TestProxiedServerLint": 6.67, - "TestProxiedServerList": 131.93, - "TestProxiedServerListComments": 1.97, - "TestProxiedServerListSortRetiresTheWalk": 0.23, - "TestProxiedServerListWatchCountsTheOffsetIntoMaxRows": 0.83, - "TestProxiedServerMemory": 90.84, - "TestProxiedServerMolBond": 267.25, - "TestProxiedServerMolBurn": 135.47, - "TestProxiedServerMolCurrent": 177.89, - "TestProxiedServerMolDistill": 55.5, - "TestProxiedServerMolLastActivity": 52.81, - "TestProxiedServerMolProgress": 41.09, - "TestProxiedServerMolReadyGated": 202.08, - "TestProxiedServerMolShow": 83.77, - "TestProxiedServerMolSquash": 73.31, - "TestProxiedServerMolStale": 52.8, - "TestProxiedServerMoleculeFrontDoorDirectServerProxyParity": 2.14, - "TestProxiedServerMoleculeFrontDoorWriteParity": 21.32, - "TestProxiedServerMultiStatementSQL": 7.5, - "TestProxiedServerNote": 21.92, - "TestProxiedServerOrphans": 4.25, - "TestProxiedServerOutageReconnectAcceptanceMatrix": 36.22, + "TestProxiedServerAssign": 16.45, + "TestProxiedServerAutoPrune": 8.6, + "TestProxiedServerBackupRefusedOnExternalTopology": 0.38, + "TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce": 8.01, + "TestProxiedServerBatchParity": 30.55, + "TestProxiedServerBlocked": 29.31, + "TestProxiedServerCascadeParity": 48.81, + "TestProxiedServerChildren": 9.53, + "TestProxiedServerCleanDatabases": 0.81, + "TestProxiedServerCleanDatabasesPurgeDropped": 2.22, + "TestProxiedServerClose": 68.94, + "TestProxiedServerClose2": 104.58, + "TestProxiedServerClose2B": 113.92, + "TestProxiedServerClose3": 50.47, + "TestProxiedServerClose4": 69.47, + "TestProxiedServerCloseB": 48.63, + "TestProxiedServerCloseBeadGate": 16.14, + "TestProxiedServerCloseConcurrent": 36.83, + "TestProxiedServerCloseSiblingBlockersFanIn": 25.27, + "TestProxiedServerComment": 24.83, + "TestProxiedServerCompactDolt": 0.2, + "TestProxiedServerCompactHistory": 1.86, + "TestProxiedServerConfig": 85.93, + "TestProxiedServerConfigB": 67.56, + "TestProxiedServerConfigConcurrent": 14.05, + "TestProxiedServerConfigSetMany": 12.34, + "TestProxiedServerContext": 9.81, + "TestProxiedServerCount": 8.05, + "TestProxiedServerCountIncludeInfra": 3.19, + "TestProxiedServerCreate": 22.17, + "TestProxiedServerCreate2": 94.47, + "TestProxiedServerCreate2B": 98.11, + "TestProxiedServerCreate2C": 70.34, + "TestProxiedServerCreateInfraTypeRoutesToWisps": 22.11, + "TestProxiedServerDeferAutoWake": 3.47, + "TestProxiedServerDeferUndefer": 9.21, + "TestProxiedServerDelete": 10.21, + "TestProxiedServerDeleteBlockedJSONSingleDoc": 9.34, + "TestProxiedServerDeleteConcurrent": 11.56, + "TestProxiedServerDeleteForceOrphans": 6.2, + "TestProxiedServerDeleteWisp": 4.71, + "TestProxiedServerDep": 91.86, + "TestProxiedServerDep2": 131.93, + "TestProxiedServerDep2B": 57.41, + "TestProxiedServerDep2C": 75.41, + "TestProxiedServerDepB": 67.92, + "TestProxiedServerDepConcurrent": 19.74, + "TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge": 8.32, + "TestProxiedServerDoltLifecycle": 0.13, + "TestProxiedServerDoltRemoteRemove": 90.8, + "TestProxiedServerDuplicates": 22.36, + "TestProxiedServerEdit": 9.97, + "TestProxiedServerEpic": 8.17, + "TestProxiedServerEventsJournal": 4.62, + "TestProxiedServerEventsJournalOffByDefault": 5.95, + "TestProxiedServerExport": 14.47, + "TestProxiedServerExternalCreate": 4.62, + "TestProxiedServerExternalMutationPolicy": 8.86, + "TestProxiedServerExternalPolicyDirectParity": 6.31, + "TestProxiedServerFindDuplicates": 22.0, + "TestProxiedServerGC": 2.77, + "TestProxiedServerGateCheck": 38.27, + "TestProxiedServerGateCheckB": 41.55, + "TestProxiedServerGateCreateInheritsRepoMetadata": 2.72, + "TestProxiedServerGateFrontDoorDirectServerProxyParity": 17.87, + "TestProxiedServerGateLifecycle": 40.73, + "TestProxiedServerGateList": 20.14, + "TestProxiedServerGatedRendering": 4.95, + "TestProxiedServerGraph": 4.6, + "TestProxiedServerHeartbeat": 4.61, + "TestProxiedServerHistory": 3.5, + "TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix": 21.43, + "TestProxiedServerHistoryRemoteSupportedFrontDoorParity": 6.74, + "TestProxiedServerHuman": 79.25, + "TestProxiedServerIfRevisionGuardAssign": 0.47, + "TestProxiedServerIfRevisionGuardClose": 0.67, + "TestProxiedServerIfRevisionGuardDelete": 0.59, + "TestProxiedServerIfRevisionGuardReopen": 1.39, + "TestProxiedServerIfRevisionGuardUpdate": 0.79, + "TestProxiedServerImport": 92.69, + "TestProxiedServerInfo": 2.71, + "TestProxiedServerInitExplicitBeadsDirIsAuthoritative": 9.07, + "TestProxiedServerInitIfMissing": 0.39, + "TestProxiedServerKV": 91.89, + "TestProxiedServerLabel": 75.59, + "TestProxiedServerLabelB": 60.23, + "TestProxiedServerLabelC": 105.9, + "TestProxiedServerLink": 26.12, + "TestProxiedServerLint": 2.66, + "TestProxiedServerList": 35.23, + "TestProxiedServerListComments": 0.81, + "TestProxiedServerListSortRetiresTheWalk": 0.08, + "TestProxiedServerListWatchCountsTheOffsetIntoMaxRows": 0.44, + "TestProxiedServerMemory": 26.25, + "TestProxiedServerMolBond": 72.46, + "TestProxiedServerMolBurn": 29.07, + "TestProxiedServerMolCurrent": 89.43, + "TestProxiedServerMolDistill": 23.19, + "TestProxiedServerMolLastActivity": 35.41, + "TestProxiedServerMolProgress": 34.63, + "TestProxiedServerMolReadyGated": 93.82, + "TestProxiedServerMolShow": 24.26, + "TestProxiedServerMolSquash": 45.59, + "TestProxiedServerMolStale": 36.09, + "TestProxiedServerMoleculeFrontDoorDirectServerProxyParity": 1.62, + "TestProxiedServerMoleculeFrontDoorWriteParity": 23.11, + "TestProxiedServerMultiStatementSQL": 5.6, + "TestProxiedServerNote": 9.88, + "TestProxiedServerOrphans": 1.81, + "TestProxiedServerOutageReconnectAcceptanceMatrix": 36.12, "TestProxiedServerPathHelpers": 0.0, - "TestProxiedServerPing": 0.76, - "TestProxiedServerPour": 90.63, - "TestProxiedServerPourRejectsEnumViolatingVar": 16.32, - "TestProxiedServerPrime": 120.62, - "TestProxiedServerPriority": 29.69, - "TestProxiedServerPromote": 137.81, - "TestProxiedServerPrune": 229.41, - "TestProxiedServerPurgeWispsPlaneRetention": 27.89, - "TestProxiedServerQuery": 32.83, - "TestProxiedServerQuick": 32.66, - "TestProxiedServerReady": 237.31, - "TestProxiedServerReady2": 159.52, - "TestProxiedServerReadyB": 279.25, - "TestProxiedServerReadyC": 232.74, - "TestProxiedServerReadyClaimConcurrent": 21.86, - "TestProxiedServerReadyConcurrent": 14.36, - "TestProxiedServerReadyWisp": 315.55, - "TestProxiedServerReassignFence": 92.12, - "TestProxiedServerReclaim": 1.41, - "TestProxiedServerRecomputeBlocked": 69.58, - "TestProxiedServerRelate": 174.16, - "TestProxiedServerReopen": 317.88, - "TestProxiedServerReopenB": 283.24, - "TestProxiedServerReopenConcurrent": 100.49, - "TestProxiedServerSQL": 4.56, - "TestProxiedServerSQLDatabaseFlag": 14.98, - "TestProxiedServerSQLStatementRouting": 2.61, - "TestProxiedServerSearch": 79.86, - "TestProxiedServerServeActivatesTheEventsJournal": 10.7, - "TestProxiedServerServeAddComment": 3.85, - "TestProxiedServerServeAddDependencies": 7.7, - "TestProxiedServerServeBatchClose": 14.42, - "TestProxiedServerServeClaim": 13.9, - "TestProxiedServerServeClaimNext": 12.27, - "TestProxiedServerServeClose": 21.8, - "TestProxiedServerServeCount": 15.2, - "TestProxiedServerServeCreate": 2.62, - "TestProxiedServerServeDelete": 5.64, - "TestProxiedServerServeEdgeCounts": 15.27, - "TestProxiedServerServeLifecycle": 0.03, - "TestProxiedServerServeModeGate": 23.75, - "TestProxiedServerServeReadParity": 6.26, - "TestProxiedServerServeReadRevision": 3.59, - "TestProxiedServerServeReadsTheJournalItJustWrote": 14.23, - "TestProxiedServerServeRefusesAStaleCheckpointWithGone": 14.62, - "TestProxiedServerServeRefusesAStreamOnADisabledJournal": 14.96, - "TestProxiedServerServeRefusesTheJournalWhenItIsDisabled": 13.62, - "TestProxiedServerServeRelatedIssues": 14.9, - "TestProxiedServerServeRelease": 4.5, - "TestProxiedServerServeRemoveDependency": 5.12, - "TestProxiedServerServeSettingsWrites": 5.08, - "TestProxiedServerServeStreamsTheJournalItIsWriting": 21.99, - "TestProxiedServerServeUpdate": 14.38, - "TestProxiedServerServeUpdateClaim": 4.97, - "TestProxiedServerSetState": 14.99, - "TestProxiedServerShow": 246.84, - "TestProxiedServerShow2": 304.16, - "TestProxiedServerShow2B": 302.98, - "TestProxiedServerShow3": 276.05, - "TestProxiedServerShow3B": 261.07, - "TestProxiedServerShowB": 297.04, - "TestProxiedServerShowWatch": 164.7, - "TestProxiedServerStale": 6.27, - "TestProxiedServerState": 189.5, - "TestProxiedServerStatus": 40.45, - "TestProxiedServerStatuses": 2.26, - "TestProxiedServerStorageClass": 337.81, - "TestProxiedServerSwarm": 8.38, - "TestProxiedServerTag": 55.84, - "TestProxiedServerTodo": 96.61, - "TestProxiedServerTypes": 4.25, - "TestProxiedServerTypesAgreeWithCreate": 37.25, - "TestProxiedServerUnclaim": 16.93, - "TestProxiedServerUnclaimIfAssigneeParity": 35.25, - "TestProxiedServerUpdate": 306.84, - "TestProxiedServerUpdate2": 285.3, - "TestProxiedServerUpdate2B": 321.6, - "TestProxiedServerUpdate2C": 247.7, - "TestProxiedServerUpdate3": 375.23, - "TestProxiedServerUpdateB": 265.81, - "TestProxiedServerUpdateC": 265.62, - "TestProxiedServerUpdateClosePolicy": 82.95, - "TestProxiedServerUpdateConcurrentAppendNotes": 7.13, - "TestProxiedServerUpdateConcurrentClaim": 12.38, - "TestProxiedServerUpdateConcurrentMetadata": 11.36, - "TestProxiedServerUpdateHooks": 52.36, - "TestProxiedServerUpdateIfGuards": 83.41, - "TestProxiedServerUpdateWisp": 268.08, - "TestProxiedServerWispCreate": 78.86, - "TestProxiedServerWispCreateRejectsEnumViolatingVar": 13.68, - "TestProxiedServerWispGC": 164.43, - "TestProxiedServerWispList": 65.42, - "TestServerModeDeferAutoWake": 11.69, - "TestServerModeServe": 0.04, - "TestServerModeServeSkipsPostRunMaintenance": 13.74, - "TestServerModeSharedServeGlobalReportsTheServedDatabase": 34.13, - "TestServerModeTypesAgreeWithCreate": 36.55 + "TestProxiedServerPing": 0.27, + "TestProxiedServerPour": 42.98, + "TestProxiedServerPourRejectsEnumViolatingVar": 10.18, + "TestProxiedServerPrime": 49.46, + "TestProxiedServerPriority": 8.17, + "TestProxiedServerPromote": 24.66, + "TestProxiedServerPrune": 56.2, + "TestProxiedServerPurgeWispsPlaneRetention": 20.91, + "TestProxiedServerQuery": 9.29, + "TestProxiedServerQuick": 21.8, + "TestProxiedServerReady": 96.02, + "TestProxiedServerReady2": 61.85, + "TestProxiedServerReadyB": 43.87, + "TestProxiedServerReadyC": 74.0, + "TestProxiedServerReadyClaimConcurrent": 7.96, + "TestProxiedServerReadyConcurrent": 9.66, + "TestProxiedServerReadyWisp": 95.28, + "TestProxiedServerReassignFence": 31.82, + "TestProxiedServerReclaim": 0.52, + "TestProxiedServerRecomputeBlocked": 56.46, + "TestProxiedServerRelate": 70.72, + "TestProxiedServerReopen": 66.48, + "TestProxiedServerReopenB": 59.36, + "TestProxiedServerReopenConcurrent": 48.76, + "TestProxiedServerSQL": 2.47, + "TestProxiedServerSQLDatabaseFlag": 13.98, + "TestProxiedServerSQLStatementRouting": 1.43, + "TestProxiedServerSearch": 33.73, + "TestProxiedServerServeActivatesTheEventsJournal": 9.88, + "TestProxiedServerServeAddComment": 1.6, + "TestProxiedServerServeAddDependencies": 5.68, + "TestProxiedServerServeBatchClose": 4.48, + "TestProxiedServerServeClaim": 5.07, + "TestProxiedServerServeClaimNext": 9.38, + "TestProxiedServerServeClose": 13.13, + "TestProxiedServerServeCount": 7.19, + "TestProxiedServerServeCreate": 1.13, + "TestProxiedServerServeDelete": 3.95, + "TestProxiedServerServeEdgeCounts": 10.32, + "TestProxiedServerServeLifecycle": 0.05, + "TestProxiedServerServeModeGate": 23.65, + "TestProxiedServerServeReadParity": 3.51, + "TestProxiedServerServeReadRevision": 3.24, + "TestProxiedServerServeReadsTheJournalItJustWrote": 16.7, + "TestProxiedServerServeRefusesAStaleCheckpointWithGone": 16.72, + "TestProxiedServerServeRefusesAStreamOnADisabledJournal": 7.45, + "TestProxiedServerServeRefusesTheJournalWhenItIsDisabled": 12.84, + "TestProxiedServerServeRelatedIssues": 7.45, + "TestProxiedServerServeRelease": 1.96, + "TestProxiedServerServeRemoveDependency": 3.87, + "TestProxiedServerServeSettingsWrites": 1.94, + "TestProxiedServerServeStreamsTheJournalItIsWriting": 24.92, + "TestProxiedServerServeUpdate": 7.19, + "TestProxiedServerServeUpdateClaim": 5.33, + "TestProxiedServerSetState": 4.87, + "TestProxiedServerShow": 77.15, + "TestProxiedServerShow2": 110.3, + "TestProxiedServerShow2B": 88.13, + "TestProxiedServerShow3": 53.11, + "TestProxiedServerShow3B": 77.05, + "TestProxiedServerShowB": 59.27, + "TestProxiedServerShowWatch": 58.3, + "TestProxiedServerStale": 3.77, + "TestProxiedServerState": 74.91, + "TestProxiedServerStatus": 7.78, + "TestProxiedServerStatuses": 1.42, + "TestProxiedServerStorageClass": 72.09, + "TestProxiedServerSwarm": 4.54, + "TestProxiedServerTag": 24.55, + "TestProxiedServerTodo": 24.62, + "TestProxiedServerTypes": 2.06, + "TestProxiedServerTypesAgreeWithCreate": 16.18, + "TestProxiedServerUnclaim": 6.9, + "TestProxiedServerUnclaimIfAssigneeParity": 23.02, + "TestProxiedServerUpdate": 99.24, + "TestProxiedServerUpdate2": 66.73, + "TestProxiedServerUpdate2B": 106.27, + "TestProxiedServerUpdate2C": 27.48, + "TestProxiedServerUpdate3": 108.59, + "TestProxiedServerUpdateB": 84.72, + "TestProxiedServerUpdateC": 118.83, + "TestProxiedServerUpdateClosePolicy": 34.08, + "TestProxiedServerUpdateConcurrentAppendNotes": 6.24, + "TestProxiedServerUpdateConcurrentClaim": 9.59, + "TestProxiedServerUpdateConcurrentMetadata": 8.05, + "TestProxiedServerUpdateHooks": 24.63, + "TestProxiedServerUpdateIfGuards": 31.82, + "TestProxiedServerUpdateWisp": 81.79, + "TestProxiedServerWispCreate": 33.64, + "TestProxiedServerWispCreateRejectsEnumViolatingVar": 10.52, + "TestProxiedServerWispGC": 56.11, + "TestProxiedServerWispList": 36.71, + "TestServerModeDeferAutoWake": 8.1, + "TestServerModeServe": 0.03, + "TestServerModeServeSkipsPostRunMaintenance": 20.92, + "TestServerModeSharedServeGlobalReportsTheServedDatabase": 21.54, + "TestServerModeTypesAgreeWithCreate": 17.63 } } diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go index 9d1867f6b..c0c61c9aa 100644 --- a/scripts/shard_manifest_test.go +++ b/scripts/shard_manifest_test.go @@ -220,7 +220,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { } } -// S3: the Bazel-only 44-shard block is not frozen like the legacy 15-shard +// S3: the Bazel-only 34-shard block is not frozen like the legacy 15-shard // block (TestShardScriptsListOnlyRealTests's B1 fix catches outright // corruption, but not a committed block that has drifted from the currently // discovered TestProxiedServer*/TestServerMode* test set, e.g. a test added, From 6d27a4f96e7fb74e2db516d7be9c80475f59dbef Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 18:30:42 +0000 Subject: [PATCH 019/151] fix(issueops): enforce the close guards on every close leg (#7425) * fix(issueops): enforce the close guards on every close leg The template read-only guard, the pin guard and the assignee authority fence (be-035) lived only in cmd/bd (validateIssueClosable), so a close through bd serve, issues:batchClose, a batch:apply close item or a library caller of Lifecycle.Close / BatchCloser / BatchApplier closed what bd close refused. They now run inside the close's own transaction in CloseIssueCheckedInTx (enforceCloseGuardsInTx), the body every checked close shares (dolt, embedded, uow, and the server behind httpapi), for a target that is not already closed: - a template refuses with *issueops.TemplateReadOnlyError (no bypass); - unless Force (CloseRequest, CloseBatchRequest, CloseItem) is set, a pinned issue refuses with *issueops.PinnedError and another actor's bead with *issueops.CloseNotAssigneeError (matches ErrNotOwner). Wire: 409 template_read_only, issue_pinned and not_assignee (holder in `assignee`, also on a batch-close outcome); the client rebuilds the typed errors. The decision is one exported pure rule over the pre-image, issueops.CheckClosable (internal/storage/issueops), which the in-tx guard calls; bd close's direct and proxied preflights call it too, ahead of gate satisfaction under the same already-closed skip, so bd close output, refusal order and exit codes are unchanged (validateIssueClosable is gone). bd close --if-revision now applies the guards too. Conformance: RunLifecycleCloseEnforcesTheCloseGuards, RunBatchCloserItemsAnswerToTheCloseGuards and RunBatchApplyCloseItemsAnswerToTheCloseGuards on dolt, embedded, uow and http, plus handler mapping tests, a CheckClosable table test, and a cmd/bd test pinning that the guards outrank gate satisfaction. Co-Authored-By: Claude Opus 5.5 * fix(httpclient): send force on the raw close; scope the close guards Review fixes for #7425 (adopt-pr bd-0h0ao4, iterations 1 and 2). The raw close keeps parity over http. httpclient.Store.CloseIssue maps the off-role raw close onto the role's closeIssue op and sent force=false. With the guards in the role, a pinned or claimed target then refused (409 issue_pinned / not_assignee) on http only: dolt, embedded and proxied run the raw close as issueops.CloseIssueInTx, which applies neither the guards nor close policy. The molecule auto-close, `bd gate resolve`, `bd todo done` and `bd human dismiss` all make that call, and its signature carries no force a caller could pass. It now sends force. Over http that also stops applying close policy to the raw close, which it did at base and no other backend does; parity is the point, so that goes too. The template guard has no bypass on the wire, so a template still refuses the raw close over http where the other backends close it; no raw caller is known to close one. TestServedRawCloseKeepsParityWithTheOtherBackends closes both pin spellings, a held bead and a parent with an open child through Store.CloseIssue and requires the template refusal; with force back to false it fails on 409 issue_pinned. The close guards stay on the close verbs. A status update that crosses into the done category (`bd update -s closed`, PATCH, a batchApply update item) still answers to close policy only, so it closes a pinned issue or another actor's bead on every backend, as it did before this PR. Guarding it changes what the update legs admit and needs its own refusal mapping on PATCH and batchApply, so it is left for a follow-up; the update.go and domain/db comments now say the exclusion is deliberate. CHANGELOG: the entry moves from Fixed to Changed as BREAKING, is scoped to the close operations, names the two paths the guards do not cover, and adds a Migration line (send force or reclaim; templates are never closable). Docs: the OpBatchCloseIssues item codes, the CloseIssueChecked and CloseIssueOptions.Force docs (storage, dolt, embeddeddolt), the oracle-a scenario strings that named the retired validateIssueClosable, and the TestCloseCheckOne_CloseGuardsOutrankGateSatisfaction doc, which claimed a proxied route it does not drive. Iteration 2: the wire contract says what force waives. The OpenAPI `force` members of CloseIssueRequest, BatchCloseRequest and ApplyCloseItem said force bypasses close policy "and nothing else"; it also waives the pin and the assignee guard, and never the template guard. The batchClose "only two answers" paragraph now says the same, and types.gen.go is regenerated (`make api-gen`). "Close guards" names the close operations instead of "every route to a close" and adds that a status update into the done category is not one, and the PATCH done-crossing paragraph says a patch is never refused `issue_pinned` or `not_assignee`. The conformance comments that quoted the old Force sentence (batch_apply_contract.go, batch_closer_contract.go, lifecycle_close_reopen_contract.go) quote the current docs, and the batch_apply failure message no longer says Force does nothing else. CHANGELOG: a `close` line of `bd batch` is named among the raw closes, and "a template is never closable" becomes "the close operations never close a template, forced or not". The storage interface CloseIssue gains a doc that states its HTTP exception. TestCloseGuardRefusalsPrintTheGuardSentenceOnEveryRoute pins the sentence `bd close` prints for each guard (template, pin, assignee) through each printer (closeDirectRefusal, closeDirectTypedRefusal, closeProxiedRefusal, closeProxiedTypedRefusal), with the refusal bare, wrapped by the use case and the SQL repository, or inside a served problem envelope. One pinned refusal also goes through closeProxiedOutcomes and closeProxiedFailures, where the stderr line and the `--json` error field must both be the sentence. Six overlay mutants fail it: closeGuardRefusal recognizing no guard, or not the pin, and each of the four printers skipping its call to closeGuardRefusal. Not done here: a CLI test that `--if-revision` exits 1 on a guard refusal, not ExitGuardMismatch. External PR #7372 carries per-id fence tests for that route (6 of 7 pass at this head); they are better absorbed with credit than rewritten. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../scripts/embedded-storage-test-shards.txt | 1 + CHANGELOG.md | 28 +++ backend/conformance/batch_apply_contract.go | 104 +++++++++- backend/conformance/batch_closer_contract.go | 100 ++++++++- .../lifecycle_close_reopen_contract.go | 196 +++++++++++++++++- backend/conformance/role_bundle_cases.go | 3 + cmd/bd/close.go | 2 +- cmd/bd/close_direct.go | 76 +++++-- cmd/bd/close_failure_report_test.go | 106 ++++++++++ cmd/bd/close_gate_test.go | 80 +++++++ cmd/bd/close_proxied_server.go | 18 +- cmd/bd/show_unit_helpers.go | 16 -- cmd/bd/show_unit_helpers_test.go | 59 ------ internal/httpapi/BUILD.bazel | 1 + internal/httpapi/apigen/types.gen.go | 15 +- internal/httpapi/batch_apply.go | 7 +- internal/httpapi/batch_close.go | 11 + internal/httpapi/close_guards_test.go | 163 +++++++++++++++ internal/httpapi/problem.go | 55 ++++- internal/httpapi/spec/openapi.v0.yaml | 111 +++++++--- .../httpapi/wireshape/testdata/golden.json | 6 + internal/httpclient/batchcloser.go | 34 ++- internal/httpclient/lifecycle.go | 18 +- .../httpclient/served_batch_apply_test.go | 4 + .../httpclient/served_batch_close_test.go | 7 + .../httpclient/served_claim_lifecycle_test.go | 69 ++++++ internal/httpclient/wire/client.go | 6 +- internal/httpclient/wire/problem.go | 17 +- internal/httpclient/wire/writes.go | 4 +- .../storage/dolt/batch_apply_contract_test.go | 5 + .../dolt/batch_closer_contract_test.go | 6 + internal/storage/dolt/issues.go | 6 +- .../lifecycle_close_reopen_contract_test.go | 3 + internal/storage/domain/db/issue.go | 8 +- .../embeddeddolt/batch_apply_contract_test.go | 3 + .../batch_closer_contract_test.go | 6 + internal/storage/embeddeddolt/issues.go | 6 +- .../lifecycle_close_reopen_contract_test.go | 3 + internal/storage/issueops/BUILD.bazel | 2 + internal/storage/issueops/close.go | 14 ++ internal/storage/issueops/close_guard.go | 112 ++++++++++ internal/storage/issueops/close_guard_test.go | 55 +++++ internal/storage/issueops/update.go | 15 +- internal/storage/storage.go | 18 +- .../storage/uow/batch_apply_contract_test.go | 3 + .../storage/uow/batch_closer_contract_test.go | 1 + .../lifecycle_close_reopen_contract_test.go | 3 + internal/validation/issue.go | 4 +- issueops/batchapplier.go | 3 +- issueops/batchcloser.go | 12 +- issueops/errors.go | 45 ++++ issueops/issueops.go | 20 +- issueops/template_errors.go | 5 +- .../harness/scenarios/enumerated.json | 4 +- 54 files changed, 1464 insertions(+), 215 deletions(-) create mode 100644 internal/httpapi/close_guards_test.go create mode 100644 internal/storage/issueops/close_guard.go create mode 100644 internal/storage/issueops/close_guard_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index da78fa452..0b2f4d259 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -417,6 +417,7 @@ 20 15 TestCreateBatchFastPathsMatchPerRowLarge_Embedded 20 15 TestDoltCommitResponseLossIsIndeterminate 20 15 TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing +20 15 TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards 20 15 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest 20 15 TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop 20 15 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b8fc52d..230391fdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -202,6 +202,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 amendment to a released migration must land as a new migration (0070+, or ignored 0028+, as of this change) so installed schemas and freshly-migrated ones cannot fork. +- **BREAKING: the close guards now hold for every close operation, not only + `bd close`.** The template read-only guard, the pin guard and the assignee + authority fence (be-035) used to be a pre-read in `cmd/bd`, so a close + through `bd serve` (`POST /v0/beads/issues/{id}:close`), `issues:batchClose`, + a `close` item of `issues:batchApply`, or a library caller of + `issueops.Lifecycle.Close`, `BatchCloser` or `BatchApplier` closed a + template, a pinned issue, or a bead another actor holds that `bd close` + refused. They now live in the role, inside the close's own transaction, on + every backend: a template refuses with `*issueops.TemplateReadOnlyError` (no + bypass), and unless `Force` is set a pinned issue refuses with + `*issueops.PinnedError` and another actor's bead with + `*issueops.CloseNotAssigneeError` (matching `ErrNotOwner`). Over HTTP they + are `409` `template_read_only`, `issue_pinned` and `not_assignee` (the + holder in `assignee`, also on a batch-close outcome), and the client + rebuilds the same typed errors. `bd close` prints the same lines and exits + as before, and `bd close --if-revision`, which skipped the guards, now + applies them too. Two paths stay outside the guards. The raw storage + `CloseIssue`, which the molecule auto-close, a `close` line of `bd batch` + and the other closes `cmd/bd` makes outside `bd close` use, runs neither the + guards nor close policy on dolt, embedded or proxied; over HTTP it now sends + `force`, so there it also stops applying close policy, and only a template + refuses. A status update into the done category (`bd update -s closed`, a + `PATCH` whose `status` is done, an `update` item of `issues:batchApply`) + answers to close policy only, as before, so it still closes a pinned issue + or another actor's bead. Migration: HTTP and library callers that close + pinned issues or beads another actor holds must send `force` (`Force` on the + request), or reclaim the bead first; the close operations never close a + template, forced or not (pour it instead). ### Fixed - **An update of a template is now refused for every caller, not only diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index 6da619b21..c2ac269d9 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -882,9 +882,12 @@ func RunBatchApplyEvaluatesExpectedAssigneeAsModified(t *testing.T, ctx context. // close it. The refusal is the assertion; the rollback of the parent it created // is what says the refusal took the request with it. // -// POSITIVE HALF: the identical request with Force lands, which is the whole of -// what Force does ("bypasses blocker and open-child close policy, and nothing -// else") and the proof that the refusal was the policy rather than the shape. +// POSITIVE HALF: the identical request with Force lands, and that is the proof +// the refusal was the policy rather than the shape. Force "bypasses blocker and +// open-child close policy, the pin and the assignee fence, and never the +// template guard" (batchapplier.go), and the parent trips no guard — the +// unforced half reached the policy, which the guards run before — so the +// policy is the only refusal Force had to waive. func RunBatchApplyClosePolicyEvaluatesAtTheCloseItem(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { t.Helper() parent := fixture.IssuePrefix + "-closepolicy-parent" @@ -924,7 +927,7 @@ func RunBatchApplyClosePolicyEvaluatesAtTheCloseItem(t *testing.T, ctx context.C t.Errorf("%s status = %q under Force, want %q", parent, got, types.StatusClosed) } if got := batchApplyColumn(t, ctx, fixture, "status", child); got != string(types.StatusOpen) { - t.Errorf("%s status = %q, want %q: Force bypasses the close policy and nothing else", child, got, types.StatusOpen) + t.Errorf("%s status = %q, want %q: Force waives the parent's close policy, it does not close the child", child, got, types.StatusOpen) } assertBatchApplyTypedEdgeCount(t, ctx, fixture, child, parent, string(publicops.DepParentChild), 1) } @@ -2561,3 +2564,96 @@ func RunBatchApplySplicesTheMetadataOfATemplateItCreates(t *testing.T, ctx conte t.Errorf("%s carries no gc.self key: the splice that finishes a template's own create must land", created) } } + +// RunBatchApplyCloseItemsAnswerToTheCloseGuards pins that a close item answers +// to the close guards Lifecycle.Close states, under CloseRequest's rules: a +// template refuses forced or not, a pin and another actor's bead refuse unless +// the item's Force is set. A refusal is the item's *ItemError carrying the +// guard's typed error, and the request is all or nothing, so the create item +// ahead of it does not land. +// +// The assignee case is AS-MODIFIED: the holder is set by an update item of the +// same request, so a guard that read the row before the request began — or not +// at all — closes it. +func RunBatchApplyCloseItemsAnswerToTheCloseGuards(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + template := fixture.IssuePrefix + "-closeguard-template" + pinned := fixture.IssuePrefix + "-closeguard-pinned" + held := fixture.IssuePrefix + "-closeguard-held" + bystander := fixture.IssuePrefix + "-closeguard-bystander" + for _, issue := range []*types.Issue{ + {ID: template, IsTemplate: true}, + {ID: pinned, Pinned: true}, + {ID: held}, + } { + issue.Title, issue.Status, issue.Priority, issue.IssueType = issue.ID, types.StatusOpen, 2, types.TypeTask + if err := fixture.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + + plan := func(target string, force bool, extra ...publicops.ApplyItem) publicops.ApplyBatchRequest { + closeItem := batchApplyClose(publicops.Ref{ID: target}) + closeItem.Close.Force = force + items := []publicops.ApplyItem{batchApplyCreate("bystander", batchApplyIssue(bystander, "must not land beside a refusal"))} + items = append(items, extra...) + return publicops.ApplyBatchRequest{Actor: "apply-closer", ForceIDPrefix: true, Items: append(items, closeItem)} + } + refused := func(label string, request publicops.ApplyBatchRequest, check func(error)) { + t.Helper() + _, err := fixture.BatchApplier.ApplyBatch(ctx, request) + var itemErr *publicops.ItemError + if !errors.As(err, &itemErr) { + t.Fatalf("%s: error = %v, want an *ItemError", label, err) + } + if itemErr.Kind != publicops.ItemClose || itemErr.Index != len(request.Items)-1 { + t.Errorf("%s: ItemError = %#v, want the close item at index %d", label, itemErr, len(request.Items)-1) + } + check(err) + assertBatchApplyRowCount(t, ctx, fixture, "issues", bystander, 0) + } + + for _, force := range []bool{false, true} { + refused("template close item", plan(template, force), func(err error) { + var refusal *publicops.TemplateReadOnlyError + if !errors.As(err, &refusal) || refusal.IssueID != template { + t.Errorf("template close item (force=%v) = %v, want *TemplateReadOnlyError naming %s", force, err, template) + } + }) + } + refused("pinned close item", plan(pinned, false), func(err error) { + var refusal *publicops.PinnedError + if !errors.As(err, &refusal) || refusal.IssueID != pinned { + t.Errorf("pinned close item = %v, want *PinnedError naming %s", err, pinned) + } + }) + assign := batchApplyUpdate(publicops.Ref{ID: held}, publicops.IssuePatch{Assignee: publicops.Field[string]{Set: true, Value: "holder"}}) + refused("held close item", plan(held, false, assign), func(err error) { + var refusal *publicops.CloseNotAssigneeError + if !errors.As(err, &refusal) { + t.Errorf("close item on a bead an earlier item assigned away = %v, want *CloseNotAssigneeError", err) + return + } + if refusal.IssueID != held || refusal.Assignee != "holder" || refusal.Actor != "apply-closer" { + t.Errorf("assignee refusal = %#v, want IssueID %q, Assignee holder, Actor apply-closer", refusal, held) + } + }) + for _, id := range []string{template, pinned, held} { + if got := batchApplyColumn(t, ctx, fixture, "status", id); got != string(types.StatusOpen) { + t.Errorf("%s status after the refusals = %q, want it still open", id, got) + } + } + + // The item's Force waives the pin and the assignee fence. + forcedPin := plan(pinned, true) + forcedPin.Items = forcedPin.Items[1:] + batchApplyMust(t, ctx, fixture, forcedPin) + forcedHeld := plan(held, true, assign) + forcedHeld.Items = forcedHeld.Items[1:] + batchApplyMust(t, ctx, fixture, forcedHeld) + for _, id := range []string{pinned, held} { + if got := batchApplyColumn(t, ctx, fixture, "status", id); got != string(types.StatusClosed) { + t.Errorf("%s status under the item's Force = %q, want %q", id, got, types.StatusClosed) + } + } +} diff --git a/backend/conformance/batch_closer_contract.go b/backend/conformance/batch_closer_contract.go index fde364126..a62b7a667 100644 --- a/backend/conformance/batch_closer_contract.go +++ b/backend/conformance/batch_closer_contract.go @@ -39,9 +39,10 @@ import ( // still reports OpenChildren (batchcloser.go:81-86); a duplicated id is one // of those at its own index, and the first occurrence's reason is what the // row keeps (batchcloser.go:28-33). -// - Force is request-wide and bypasses only blocker and open-child policy; -// the per-item precondition its clause also names is a category the request -// type keeps EMPTY (batchcloser.go:43-53). +// - Force is request-wide and bypasses blocker and open-child policy, the +// pin and the assignee fence, never the template guard; the per-item +// precondition its clause also names is a category the request type keeps +// EMPTY (batchcloser.go:43-55). // - ClaimNext runs after the closes, inside the same transaction, and only // when at least one item closed (batchcloser.go:55-70, 100-103, 143-146). // - LANDED means CHANGED: an all-idempotent batch earns no claim and records @@ -1397,3 +1398,96 @@ func requireBatchCloserWisps(t *testing.T, fixture BatchCloserFixture) { t.Skipf("fixture has no CreateWisp: this backend cannot seed the ephemeral plane, so the ephemeral rules are UNPINNED here") } } + +// RunBatchCloserItemsAnswerToTheCloseGuards pins that every item of a batch +// answers to the close guards Lifecycle.Close states — template, pin, assignee +// — as a PER-ITEM refusal carrying the same typed error, while the survivors +// still commit; and that the request-wide Force waives the pin and the +// assignee fence and never the template. +// +// The guarded items sit between two plain ones, so an implementation that +// aborted the batch on a guard, or skipped the guard for items after the +// first, fails here rather than in `bd close a b c`. +func RunBatchCloserItemsAnswerToTheCloseGuards(t *testing.T, ctx context.Context, fixture BatchCloserFixture) { + t.Helper() + first := fixture.IssuePrefix + "-guard-first" + template := fixture.IssuePrefix + "-guard-template" + pinned := fixture.IssuePrefix + "-guard-pinned" + held := fixture.IssuePrefix + "-guard-held" + last := fixture.IssuePrefix + "-guard-last" + seedBatchCloserIssue(t, ctx, fixture, first) + seedBatchCloserIssue(t, ctx, fixture, last) + for _, issue := range []*types.Issue{ + {ID: template, IsTemplate: true}, + {ID: pinned, Status: types.StatusPinned}, + {ID: held, Assignee: "holder"}, + } { + issue.Title, issue.Priority, issue.IssueType = issue.ID, 2, types.TypeTask + if issue.Status == "" { + issue.Status = types.StatusOpen + } + if err := fixture.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + + result, err := fixture.Closer.CloseBatch(ctx, publicops.CloseBatchRequest{ + Actor: "closer", + Items: []publicops.BatchCloseItem{{IssueID: first}, {IssueID: template}, {IssueID: pinned}, {IssueID: held}, {IssueID: last}}, + }) + if err != nil { + t.Fatalf("CloseBatch over guarded items: %v, want per-item refusals", err) + } + if len(result.Outcomes) != 5 { + t.Fatalf("outcomes = %d, want 5", len(result.Outcomes)) + } + for _, i := range []int{0, 4} { + if result.Outcomes[i].Err != nil || !result.Outcomes[i].Changed { + t.Errorf("outcome %d (%s) = %+v, want the plain item to land", i, result.Outcomes[i].IssueID, result.Outcomes[i]) + } + } + var templateRefusal *publicops.TemplateReadOnlyError + if !errors.As(result.Outcomes[1].Err, &templateRefusal) || templateRefusal.IssueID != template { + t.Errorf("template item = %v, want *TemplateReadOnlyError naming %s", result.Outcomes[1].Err, template) + } + var pinRefusal *publicops.PinnedError + if !errors.As(result.Outcomes[2].Err, &pinRefusal) || pinRefusal.IssueID != pinned { + t.Errorf("pinned item = %v, want *PinnedError naming %s", result.Outcomes[2].Err, pinned) + } + var assigneeRefusal *publicops.CloseNotAssigneeError + if !errors.As(result.Outcomes[3].Err, &assigneeRefusal) { + t.Errorf("held item = %v, want *CloseNotAssigneeError", result.Outcomes[3].Err) + } else if assigneeRefusal.IssueID != held || assigneeRefusal.Assignee != "holder" || assigneeRefusal.Actor != "closer" { + t.Errorf("held item refusal = %#v, want IssueID %q, Assignee holder, Actor closer", assigneeRefusal, held) + } + for _, i := range []int{1, 2, 3} { + if result.Outcomes[i].Issue != nil { + t.Errorf("refused outcome %d carries an Issue snapshot, want nil beside Err", i) + } + } + assertBatchCloserStatus(t, ctx, fixture, first, types.StatusClosed) + assertBatchCloserStatus(t, ctx, fixture, last, types.StatusClosed) + assertBatchCloserStatus(t, ctx, fixture, template, types.StatusOpen) + assertBatchCloserStatus(t, ctx, fixture, pinned, types.StatusPinned) + assertBatchCloserStatus(t, ctx, fixture, held, types.StatusOpen) + + forced, err := fixture.Closer.CloseBatch(ctx, publicops.CloseBatchRequest{ + Actor: "closer", + Items: []publicops.BatchCloseItem{{IssueID: template}, {IssueID: pinned}, {IssueID: held}}, + Force: true, + }) + if err != nil { + t.Fatalf("forced CloseBatch over guarded items: %v", err) + } + if !errors.Is(forced.Outcomes[0].Err, publicops.ErrTemplateReadOnly) { + t.Errorf("forced template item = %v, want ErrTemplateReadOnly: force never waives read-only", forced.Outcomes[0].Err) + } + for _, i := range []int{1, 2} { + if forced.Outcomes[i].Err != nil || !forced.Outcomes[i].Changed { + t.Errorf("forced outcome %d (%s) = %+v, want force to bypass the guard", i, forced.Outcomes[i].IssueID, forced.Outcomes[i]) + } + } + assertBatchCloserStatus(t, ctx, fixture, template, types.StatusOpen) + assertBatchCloserStatus(t, ctx, fixture, pinned, types.StatusClosed) + assertBatchCloserStatus(t, ctx, fixture, held, types.StatusClosed) +} diff --git a/backend/conformance/lifecycle_close_reopen_contract.go b/backend/conformance/lifecycle_close_reopen_contract.go index 2395cfa3c..c11556181 100644 --- a/backend/conformance/lifecycle_close_reopen_contract.go +++ b/backend/conformance/lifecycle_close_reopen_contract.go @@ -846,11 +846,12 @@ func RunLifecycleCloseAndReopenSpanTheConfiguredDoneCategory(t *testing.T, ctx c // RunLifecycleExpectedVersionIsCheckedBeforeTheNoOps pins the ORDERING clause // both requests spell out. CloseRequest.ExpectedVersion "requires the current -// row version to match and is checked before an idempotent close" -// (issueops/issueops.go:295-300); ReopenRequest.ExpectedVersion is "checked -// before a non-done no-op" (:272-275); and Close's own doc adds "ExpectedVersion +// row version to match, and is checked BEFORE the idempotent close" +// (issueops/issueops.go:362-373); ReopenRequest.ExpectedVersion is "checked +// before a non-done no-op" (:390-391); and Close's own doc adds "ExpectedVersion // is checked first, including for an idempotent close", with Force bypassing -// "blocker and open-child policy" and nothing else (:364-365). +// "the pin, the assignee fence, and blocker and open-child policy" and nothing +// else (:485-498). // // The dangerous shape is not a stale version on a live mutation — it is a stale // version on the request that would have done nothing anyway. An implementation @@ -1383,13 +1384,14 @@ func RunLifecycleCloseSettlesItsTransitiveAndCrossPlaneDependers(t *testing.T, c // RunLifecycleCloseAdmitsATransitivelyBlockedTarget uses clears the first and // not the second, and closing the child to clear the second would take the // second subject with it, since a closed row cannot be blocked and so has -// nothing left to flip. Force waives close policy and nothing else — -// CloseRequest.Force "bypasses only blocker and open-child close policy" and -// "never bypasses validation, ExpectedVersion, or lifecycle rules" -// (issueops/issueops.go:310-311) — so it is the shape that keeps both subjects -// observable, and it is not exotic: it is the same forced close of a -// blocked issue that RunLifecycleCloseIsIdempotentOnAClosedRowThatStillLooksBlocked -// is built on. +// nothing left to flip. Force waives close policy, the pin and the assignee +// fence, and nothing else — CloseRequest.Force "bypasses blocker and +// open-child close policy and the two close guards a force has always waived +// in bd" and "never bypasses the template guard, validation, ExpectedVersion, +// or lifecycle rules" (issueops/issueops.go:357-360) — so it is the shape that +// keeps both subjects observable, and it is not exotic: it is the same forced +// close of a blocked issue that +// RunLifecycleCloseIsIdempotentOnAClosedRowThatStillLooksBlocked is built on. // // All three legs reach ONE body here (internal/storage/issueops.closeIssueInTx), // so this is a wrapper and engine check rather than a third vote, exactly as @@ -1822,3 +1824,175 @@ func (c *lifecycleCloseReopenEventCounter) assertNoneAdded(t *testing.T, label s } c.total = total } + +// RunLifecycleCloseEnforcesTheCloseGuards pins the three close guards +// issueops.Lifecycle.Close states before close policy — template read-only, +// the pin, and the assignee authority fence (be-035) — on the role itself, so +// every caller of it is held to them and not only `bd close`, which used to +// apply them as a CLI pre-read the served and library routes skipped. +// +// Each refusal is asserted at the level a caller acts on: the typed error with +// its fields, its sentinel, and the sentence bd prints — read off the typed +// error, because a served refusal wraps it in the problem envelope — plus the +// row and the event stream left untouched. Force is asserted as the bypass for +// exactly the two guards it has always waived, and NOT for the template. +// +// The controls are load-bearing too: an unassigned issue and one held by the +// closing actor under a different spelling both close unforced, so a guard +// that refused every assigned row, or compared verbatim, fails here. A wisp +// case proves the guard reads the plane the close writes. A pinned issue with +// an open child reads the PIN's refusal, pinning the order (guards before +// policy). And a forced close of a boolean-pinned issue leaves the pin set, so +// the plain retry must stay the idempotent no-op (ga-ktn9pe.4.8) rather than +// refuse on the residue. +func RunLifecycleCloseEnforcesTheCloseGuards(t *testing.T, ctx context.Context, fixture LifecycleCloseReopenFixture) { + t.Helper() + + template := fixture.IssuePrefix + "-lcg-template" + statusPinned := fixture.IssuePrefix + "-lcg-status-pinned" + flagPinned := fixture.IssuePrefix + "-lcg-flag-pinned" + held := fixture.IssuePrefix + "-lcg-held" + respelled := fixture.IssuePrefix + "-lcg-respelled" + unassigned := fixture.IssuePrefix + "-lcg-unassigned" + pinnedParent := fixture.IssuePrefix + "-lcg-pinned-parent" + pinnedChild := fixture.IssuePrefix + "-lcg-pinned-child" + wisp := fixture.IssuePrefix + "-lcg-wisp-held" + + seed := func(issue *types.Issue) { + t.Helper() + issue.Title, issue.Priority, issue.IssueType = issue.ID, 2, types.TypeTask + if issue.Status == "" { + issue.Status = types.StatusOpen + } + if err := fixture.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + seed(&types.Issue{ID: template, IsTemplate: true}) + seed(&types.Issue{ID: statusPinned, Status: types.StatusPinned}) + seed(&types.Issue{ID: flagPinned, Pinned: true}) + seed(&types.Issue{ID: held, Assignee: "holder"}) + seed(&types.Issue{ID: respelled, Assignee: "gastown.closer"}) + seed(&types.Issue{ID: unassigned}) + seed(&types.Issue{ID: pinnedParent, Pinned: true}) + seed(&types.Issue{ID: pinnedChild}) + lifecycleCloseReopenSeedEdge(t, ctx, fixture, pinnedChild, pinnedParent, types.DepParentChild) + + refuses := func(id string, force bool, label string, check func(error)) { + t.Helper() + before := lifecycleCloseReopenReadRow(t, ctx, fixture, id) + events := newLifecycleCloseReopenEventCounter(t, ctx, fixture, id) + _, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: "closer", IssueID: id, Force: force}) + check(err) + lifecycleCloseReopenAssertRow(t, ctx, fixture, id, "after the "+label+" refusal", before) + events.assertNoneAdded(t, label+" refusal") + } + + // The template refuses, forced or not: no flag waives read-only. + for _, force := range []bool{false, true} { + refuses(template, force, "template", func(err error) { + var refusal *publicops.TemplateReadOnlyError + if !errors.As(err, &refusal) { + t.Fatalf("close of template %s (force=%v): err = %v, want *TemplateReadOnlyError", template, force, err) + } + if refusal.IssueID != template || !errors.Is(err, publicops.ErrTemplateReadOnly) { + t.Errorf("template refusal = %#v (%v), want IssueID %q matching ErrTemplateReadOnly", refusal, err, template) + } + want := "cannot modify template " + template + ": templates are read-only; use 'bd mol pour' to create a work item" + if refusal.Error() != want { + t.Errorf("template refusal reads %q, want %q", refusal.Error(), want) + } + }) + } + + // Both pin spellings refuse unforced (ga-z3vht). + for _, id := range []string{statusPinned, flagPinned} { + refuses(id, false, "pin", func(err error) { + var refusal *publicops.PinnedError + if !errors.As(err, &refusal) { + t.Fatalf("unforced close of pinned %s: err = %v, want *PinnedError", id, err) + } + if refusal.IssueID != id || !errors.Is(err, publicops.ErrPinned) { + t.Errorf("pin refusal = %#v (%v), want IssueID %q matching ErrPinned", refusal, err, id) + } + if want := "cannot modify pinned issue " + id + " (use --force to override)"; refusal.Error() != want { + t.Errorf("pin refusal reads %q, want %q", refusal.Error(), want) + } + }) + } + + // Another actor's bead refuses unforced, naming holder and actor. + refuses(held, false, "assignee", func(err error) { + var refusal *publicops.CloseNotAssigneeError + if !errors.As(err, &refusal) { + t.Fatalf("unforced close of %s held by another actor: err = %v, want *CloseNotAssigneeError", held, err) + } + if refusal.IssueID != held || refusal.Assignee != "holder" || refusal.Actor != "closer" { + t.Errorf("assignee refusal = %#v, want IssueID %q, Assignee holder, Actor closer", refusal, held) + } + if !errors.Is(err, publicops.ErrNotOwner) { + t.Errorf("assignee refusal %v does not match ErrNotOwner", err) + } + if want := `cannot close ` + held + `: assignee is "holder", actor is "closer"; reclaim or use --force to override`; refusal.Error() != want { + t.Errorf("assignee refusal reads %q, want %q", refusal.Error(), want) + } + }) + + // The guard runs before close policy: a pinned parent with an open child + // reads the pin, not the open children. + refuses(pinnedParent, false, "pin-before-policy", func(err error) { + if !errors.Is(err, publicops.ErrPinned) { + t.Fatalf("unforced close of pinned parent %s with an open child: err = %v, want ErrPinned first", pinnedParent, err) + } + }) + + // The controls: an unassigned bead and the actor's own bead under another + // spelling of its identity close unforced. + for _, tc := range []struct{ id, actor string }{{unassigned, "closer"}, {respelled, "gastown_closer"}} { + result, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: tc.actor, IssueID: tc.id}) + if err != nil { + t.Fatalf("unforced close of %s by %s: %v, want it to land", tc.id, tc.actor, err) + } + if !result.Changed || result.Issue.Status != types.StatusClosed { + t.Errorf("unforced close of %s = %#v, want a committed close", tc.id, result) + } + } + + // Force bypasses the pin and the assignee fence. + for _, id := range []string{statusPinned, flagPinned, held} { + result, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: "closer", IssueID: id, Force: true}) + if err != nil { + t.Fatalf("forced close of %s: %v, want force to bypass the guard", id, err) + } + if !result.Changed || result.Issue.Status != types.StatusClosed { + t.Errorf("forced close of %s = %#v, want a committed close", id, result) + } + } + + // The forced close left pinned=true on the flag-pinned row; the plain retry + // is the idempotent re-close, not a pin refusal. + retry, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: "closer", IssueID: flagPinned}) + if err != nil { + t.Fatalf("plain re-close of the already-closed, still-pinned %s: %v, want the idempotent no-op", flagPinned, err) + } + if retry.Changed { + t.Errorf("plain re-close of %s reported Changed = true, want the no-op", flagPinned) + } + + // The guard reads the plane the close writes. + if err := fixture.CreateWisp(ctx, &types.Issue{ + ID: wisp, Title: wisp, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, Assignee: "holder", Ephemeral: true, + }, "seed"); err != nil { + t.Fatalf("seed wisp %s: %v", wisp, err) + } + if _, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: "closer", IssueID: wisp}); !errors.As(err, new(*publicops.CloseNotAssigneeError)) { + t.Fatalf("unforced close of wisp %s held by another actor: err = %v, want *CloseNotAssigneeError", wisp, err) + } + var status string + if err := fixture.QueryScalar(ctx, "SELECT status FROM wisps WHERE id = ?", []any{wisp}, &status); err != nil { + t.Fatalf("read wisp %s status: %v", wisp, err) + } + if types.Status(status) != types.StatusOpen { + t.Errorf("wisp %s status after the refusal = %q, want it still open", wisp, status) + } +} diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 059a50056..27f0d762f 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -60,6 +60,7 @@ var roleContractCases = []roleContract{ RunBatchApplyRefusesAnUnusableRequest, RunBatchApplyUpdateItemsRefuseATemplate, RunBatchApplySplicesTheMetadataOfATemplateItCreates, + RunBatchApplyCloseItemsAnswerToTheCloseGuards, ), roleCases("BatchCloser", "BatchCloser()", oncePerRole, @@ -84,6 +85,7 @@ var roleContractCases = []roleContract{ RunBatchCloserAllRefusedBatchRecordsNoHistory, RunBatchCloserDoesNotMutateTheCallerRequest, RunBatchCloserSettlesTheDependersOfWhatItClosed, + RunBatchCloserItemsAnswerToTheCloseGuards, ), roleCases("BatchCreator", "BatchCreator()", oncePerRole, @@ -442,6 +444,7 @@ var roleContractCases = []roleContract{ RunLifecycleCloseSettlesTheClosedRowItselfAndItsChild, RunLifecycleCloseOnASpawnersLastChildSatisfiesAWaitsForGate, RunLifecycleReopenReblocksItsDependers, + RunLifecycleCloseEnforcesTheCloseGuards, ), // The accessor-reachable half of Lifecycle.Create, moved out of the staging diff --git a/cmd/bd/close.go b/cmd/bd/close.go index 5a28910a2..07eec5fa2 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -185,7 +185,7 @@ the flags appear in the command line.`, // The typed error, not the decorated display line: the --force // hint closeDirectRefusal appends is advice for a human reader, // and the id already has its own field. - failures = append(failures, closeIDFailure{ID: id, Error: res.Err.Error()}) + failures = append(failures, closeIDFailure{ID: id, Error: closeDirectTypedRefusal(res.Err)}) continue } diff --git a/cmd/bd/close_direct.go b/cmd/bd/close_direct.go index 47890945e..79a91c59a 100644 --- a/cmd/bd/close_direct.go +++ b/cmd/bd/close_direct.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/steveyegge/beads/internal/storage" + storeissueops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -36,12 +37,16 @@ type closeDirectPlan struct { // closeDirectPreflight applies `bd close`'s own close policy to every resolved // argument, and hands the batch only the items that survived. // -// Neither check is library policy — the template/pin/assignee fence and gate -// satisfaction read the issue and nothing else, and the role has no vocabulary -// for either — which is why they stay in cmd/bd. It is closeProxiedRunPreflight's +// The one check left here is gate satisfaction. It is closeProxiedRunPreflight's // job on this route. // -// The open-children guard is NOT here, and that is the point: the engine +// The close guards — template, pin, assignee — are NOT here: they are the +// role's (issueops.Lifecycle.Close), enforced inside the close's own +// transaction on every route to a close, and they come back as the item's +// typed refusal, which closeDirectRefusal prints as the sentence this route +// always printed. +// +// The open-children guard is NOT here either, and that is the point: the engine // enforces it inside the close's own transaction, refusing with a // CloseOpenChildrenError whose message is the one this route used to compose // by hand. Keeping a copy here would be a second implementation of a rule the @@ -67,21 +72,16 @@ func closeDirectPreflight(results []*RoutedResult, resolvedIDs, reasons []string // closeDirectCheckOne returns one argument's refusal, or "" when it may go to // the batch. st is the store that owns the issue. func closeDirectCheckOne(id string, issue *types.Issue, st storage.DoltStorage, force bool) string { - // Close validation guards a state change; a row already at literal - // StatusClosed has none to guard, so skip it and let the re-close reach the - // engine as the idempotent no-op it has always been (ga-ktn9pe.4.8). - // Without this, a forced close of a boolean-pinned bead leaves pinned=true - // (the close write never touches the column — deliberately, it is the - // deletion-protection flag bd gc/purge/cleanup honor) and the plain retry - // hits NotPinned and exits nonzero, which strands the molecule auto-close - // the unchanged branch re-drives. Only a literal closed status qualifies: - // reaching a configured done status is still a real close, mirroring the - // engine's isClosedInTx. The snapshot only decides whether validation runs, - // never what is written — the engine's in-transaction `status != closed` - // guard remains the authority on whether the close is a no-op, so a - // concurrent close still converges. Mirrored in closeProxiedCheckOne. + // The close guards' own rule (storeissueops.CheckClosable), applied to the + // resolved pre-image ahead of gate satisfaction so a refusal that trips both + // reads the guard's sentence, as it always has. The role applies the same + // rule inside the close transaction, which stays the authority. A row + // already at literal StatusClosed has no state change to guard: the re-close + // reaches the engine as the idempotent no-op it has always been + // (ga-ktn9pe.4.8) — a forced close of a pinned bead leaves pinned=true, and + // the plain retry must not then refuse. Mirrored in closeProxiedCheckOne. if issue == nil || issue.Status != types.StatusClosed { - if err := validateIssueClosable(id, issue, currentActor(), force); err != nil { + if err := storeissueops.CheckClosable(id, issue, currentActor(), force); err != nil { return err.Error() } } @@ -210,6 +210,9 @@ func closeDirectCloseBatch(ctx context.Context, st storage.DoltStorage, request // line this route used to compose before the guard moved into the transaction. // Everything else is an unexpected failure and is reported against the id. func closeDirectRefusal(id string, err error) string { + if guard, ok := closeGuardRefusal(err); ok { + return guard + } switch { case errors.Is(err, storage.ErrCloseBlocked): return fmt.Sprintf("%v (use --force to override)", err) @@ -219,3 +222,40 @@ func closeDirectRefusal(id string, err error) string { return fmt.Sprintf("Error closing %s: %v", id, err) } } + +// closeGuardRefusal returns a close guard's own sentence when err is one of the +// three refusals the role raises before close policy — a template, a pin, or a +// bead someone else holds — and reports whether it was. +// +// The sentence is the typed error's, found with errors.As rather than taken +// from err itself, because err may have traveled: the unit-of-work route wraps +// it in the use case's and repository's prefixes, and a served one in the +// problem envelope. Each guard's message already names its subject and its +// --force hint, so it prints bare — the line `bd close` printed when the guard +// was a CLI pre-read, on every route. +func closeGuardRefusal(err error) (string, bool) { + var ( + template *issueops.TemplateReadOnlyError + pinned *issueops.PinnedError + notAssignee *issueops.CloseNotAssigneeError + ) + switch { + case errors.As(err, &template): + return template.Error(), true + case errors.As(err, &pinned): + return pinned.Error(), true + case errors.As(err, ¬Assignee): + return notAssignee.Error(), true + } + return "", false +} + +// closeDirectTypedRefusal is the reason a --json consumer reads for one refused +// id on this route: the typed error, with a close guard's sentence taken from +// the guard itself so a served refusal reads as a local one does. +func closeDirectTypedRefusal(err error) string { + if guard, ok := closeGuardRefusal(err); ok { + return guard + } + return err.Error() +} diff --git a/cmd/bd/close_failure_report_test.go b/cmd/bd/close_failure_report_test.go index cb05c024e..58879c84d 100644 --- a/cmd/bd/close_failure_report_test.go +++ b/cmd/bd/close_failure_report_test.go @@ -279,3 +279,109 @@ func TestCloseProxiedFailuresKeepPolicyRefusalsAndTypedOrder(t *testing.T) { t.Errorf("failures[0].Error = %q, want the display line %q as the fallback", got[0].Error, policyRefusal) } } + +// closeGuardServedEnvelope stands in for the problem envelope a served refusal +// arrives in (wire.ProblemError): a message of its own that never repeats the +// guard's sentence, with the typed refusal reachable only through Unwrap. +type closeGuardServedEnvelope struct{ inner error } + +func (e closeGuardServedEnvelope) Error() string { + return "closeIssue: bd serve at http://127.0.0.1:1 answered 409 issue_pinned: this issue is pinned" +} + +func (e closeGuardServedEnvelope) Unwrap() error { return e.inner } + +// The close guards are the role's now, so they reach the CLI as typed refusals +// — and the line a person reads, and the failed[].error a --json consumer +// reads, must still be the sentence `bd close` printed when they were a CLI +// pre-read: on both routes, and however far the refusal traveled. Each guard's +// message already names its subject and its --force hint, so every printer +// hands it back BARE, with no "Error closing :" prefix and no second hint. +// +// The "want" column is transcribed from the pre-read's validators +// (validation.NotTemplate, NotPinned, AssigneeMatches before the move), so the +// sentences themselves are pinned, not only their relationship to the typed +// errors. The bare rows catch the decorating printers; the wrapped rows are +// the ones that catch the typed printers, which would otherwise hand back the +// whole chain. +func TestCloseGuardRefusalsPrintTheGuardSentenceOnEveryRoute(t *testing.T) { + guards := []struct { + name string + err error + want string + }{ + { + name: "template", + err: &issueops.TemplateReadOnlyError{IssueID: "gr-1"}, + want: "cannot modify template gr-1: templates are read-only; use 'bd mol pour' to create a work item", + }, + { + name: "pinned", + err: &issueops.PinnedError{IssueID: "gr-1"}, + want: "cannot modify pinned issue gr-1 (use --force to override)", + }, + { + name: "not assignee", + err: &issueops.CloseNotAssigneeError{IssueID: "gr-1", Assignee: "alice", Actor: "bob"}, + want: `cannot close gr-1: assignee is "alice", actor is "bob"; reclaim or use --force to override`, + }, + } + // How the refusal reaches the printer: as the engine minted it (the direct + // route), wrapped by the use case and the SQL repository (the proxied route; + // see TestCloseProxiedTypedRefusalUnwrapsTheRoleWrappers), or inside a + // served problem envelope. + travels := []struct { + name string + wrap func(error) error + }{ + {name: "bare", wrap: func(err error) error { return err }}, + {name: "wrapped by the role", wrap: func(err error) error { + return fmt.Errorf("close gr-1: %w", fmt.Errorf("db: IssueSQLRepository.CloseChecked gr-1: %w", err)) + }}, + {name: "in a served envelope", wrap: func(err error) error { return closeGuardServedEnvelope{inner: err} }}, + } + printers := []struct { + name string + print func(error) string + }{ + {name: "closeDirectRefusal", print: func(err error) string { return closeDirectRefusal("gr-1", err) }}, + {name: "closeDirectTypedRefusal", print: closeDirectTypedRefusal}, + {name: "closeProxiedRefusal", print: func(err error) string { return closeProxiedRefusal("gr-1", err) }}, + {name: "closeProxiedTypedRefusal", print: closeProxiedTypedRefusal}, + } + + for _, g := range guards { + for _, tr := range travels { + for _, p := range printers { + t.Run(g.name+"/"+tr.name+"/"+p.name, func(t *testing.T) { + if got := p.print(tr.wrap(g.err)); got != g.want { + t.Errorf("%s = %q, want the guard's own sentence %q", p.name, got, g.want) + } + }) + } + } + } + + // And through the proxied route's own bookkeeping, which is where the two + // printers meet: the stderr line and failed[].error are the same sentence, + // because the hint a guard needs is already in it. + pinned := guards[1] + args := []string{"gr-1"} + pre := closeProxiedPreflight{ + items: []issueops.BatchCloseItem{{IssueID: "gr-1", Reason: "done"}}, + itemArgs: []int{0}, + before: map[string]*types.Issue{}, + errors: make([]string, len(args)), + failureErrors: make([]string, len(args)), + } + closeProxiedOutcomes(&pre, issueops.CloseBatchResult{ + Outcomes: []issueops.CloseOutcome{{IssueID: "gr-1", Err: travels[1].wrap(pinned.err)}}, + }) + failures := closeProxiedFailures(&pre, args) + if len(failures) != 1 || failures[0].Error != pinned.want { + t.Errorf("failures = %+v, want one entry whose error is %q", failures, pinned.want) + } + if pre.errors[0] != pinned.want { + t.Errorf("stderr line = %q, want %q", pre.errors[0], pinned.want) + } +} diff --git a/cmd/bd/close_gate_test.go b/cmd/bd/close_gate_test.go index f935de4d3..542fa7ec2 100644 --- a/cmd/bd/close_gate_test.go +++ b/cmd/bd/close_gate_test.go @@ -6,6 +6,7 @@ import ( "errors" "strings" "testing" + "time" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" @@ -350,3 +351,82 @@ func TestCloseBeadGateGetter_RouteSelection(t *testing.T) { t.Error("direct mode: the getter does not read the gate's own store") } } + +// TestCloseCheckOne_CloseGuardsOutrankGateSatisfaction pins `bd close`'s +// refusal order on the direct preflight route (closeDirectCheckOne): the close +// guards (storeissueops.CheckClosable, the role's own rule) answer before gate +// satisfaction, so a pinned, held or template gate whose condition is unmet +// prints the guard's sentence, as it always has. Force waives the pin and the +// holder (and the gate check with them) but never the template, and a row +// already closed skips the guards (ga-ktn9pe.4.8). closeProxiedCheckOne makes +// the same call in the same order; it resolves the row through a unit of work, +// so this test does not drive it. +func TestCloseCheckOne_CloseGuardsOutrankGateSatisfaction(t *testing.T) { + actorMu.Lock() + savedActor, savedPending := actor, actorGitFallbackPending + actor, actorGitFallbackPending = "alice", false + actorMu.Unlock() + t.Cleanup(func() { + actorMu.Lock() + actor, actorGitFallbackPending = savedActor, savedPending + actorMu.Unlock() + }) + + // An unexpired timer gate: checkGateSatisfaction refuses it unforced. + gate := func(mut func(*types.Issue)) *types.Issue { + issue := &types.Issue{ + ID: "bd-g1", + IssueType: "gate", + AwaitType: "timer", + Status: types.StatusOpen, + CreatedAt: time.Now(), + Timeout: time.Hour, + } + mut(issue) + return issue + } + const gateRefusal = "cannot close bd-g1: gate condition not satisfied" + cases := []struct { + name string + issue *types.Issue + force bool + want string + }{ + {"pinned", gate(func(i *types.Issue) { i.Pinned = true }), false, + "cannot modify pinned issue bd-g1 (use --force to override)"}, + {"pinned status", gate(func(i *types.Issue) { i.Status = types.StatusPinned }), false, + "cannot modify pinned issue bd-g1 (use --force to override)"}, + {"template", gate(func(i *types.Issue) { i.IsTemplate = true }), false, + "cannot modify template bd-g1: templates are read-only; use 'bd mol pour' to create a work item"}, + {"forced template", gate(func(i *types.Issue) { i.IsTemplate = true }), true, + "cannot modify template bd-g1: templates are read-only; use 'bd mol pour' to create a work item"}, + {"held", gate(func(i *types.Issue) { i.Assignee = "bob" }), false, + `cannot close bd-g1: assignee is "bob", actor is "alice"; reclaim or use --force to override`}, + {"unguarded gate", gate(func(*types.Issue) {}), false, gateRefusal}, + {"forced pin", gate(func(i *types.Issue) { i.Pinned = true; i.Assignee = "bob" }), true, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + check := func(route, got string) { + t.Helper() + if tc.want == gateRefusal { + if !strings.HasPrefix(got, gateRefusal) { + t.Fatalf("%s refusal = %q, want the gate refusal", route, got) + } + return + } + if got != tc.want { + t.Fatalf("%s refusal = %q, want %q", route, got, tc.want) + } + } + check("direct", closeDirectCheckOne("bd-g1", tc.issue, nil, tc.force)) + }) + } + + // The already-closed skip: a forced close leaves pinned=true behind, and + // the plain re-close must reach the engine as the idempotent no-op. + residue := &types.Issue{ID: "bd-c1", Status: types.StatusClosed, Pinned: true, Assignee: "bob"} + if got := closeDirectCheckOne("bd-c1", residue, nil, false); got != "" { + t.Fatalf("closed residue refusal = %q, want none", got) + } +} diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index e7a791d4f..cf94bba83 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -12,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/audit" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" + storeissueops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" @@ -294,13 +295,12 @@ func closeProxiedCheckOne(ctx context.Context, uw uow.UnitOfWork, id string, in return fmt.Sprintf("Error resolving %s: %v", id, err), nil } - // Mirrors the ordering in closeDirectCheckOne (ga-ktn9pe.4.8): a row already at - // literal StatusClosed has no state change for close validation to guard, so - // the re-close skips it and reaches the engine as the idempotent no-op it has - // always been. Both close paths must agree here — diverging is the defect - // class #5217 closed. + // The close guards' own rule, ahead of gate satisfaction and under the same + // already-closed skip as closeDirectCheckOne (ga-ktn9pe.4.8); both close + // paths must agree here (#5217). The role re-applies it inside the close + // transaction, which stays the authority. if current.Status != types.StatusClosed { - if err := validateIssueClosable(id, current, currentActor(), in.force); err != nil { + if err := storeissueops.CheckClosable(id, current, currentActor(), in.force); err != nil { return err.Error(), nil } } @@ -419,6 +419,9 @@ func closeProxiedFailures(pre *closeProxiedPreflight, args []string) []closeIDFa // not noise, it is the only thing that says where an unexpected failure came // from, and there is no engine sentence to converge on. func closeProxiedTypedRefusal(err error) string { + if guard, ok := closeGuardRefusal(err); ok { + return guard + } for _, sentinel := range []error{storage.ErrCloseBlocked, storage.ErrCloseOpenChildren} { if !errors.Is(err, sentinel) { continue @@ -445,6 +448,9 @@ func closeProxiedTypedRefusal(err error) string { // reading the message, which is the point of the outcome carrying a typed // error at all. func closeProxiedRefusal(id string, err error) string { + if guard, ok := closeGuardRefusal(err); ok { + return guard + } switch { case errors.Is(err, storage.ErrCloseBlocked): return fmt.Sprintf("%v (use --force to override)", err) diff --git a/cmd/bd/show_unit_helpers.go b/cmd/bd/show_unit_helpers.go index 74c0e9705..c866d8672 100644 --- a/cmd/bd/show_unit_helpers.go +++ b/cmd/bd/show_unit_helpers.go @@ -25,22 +25,6 @@ func validateIssueUpdatable(id string, issue *types.Issue) error { return validation.NotTemplate()(id, issue) } -// validateIssueClosable checks if an issue can be closed. -// Uses the centralized validation package for consistency. -// -// actor is the current actor identity (may be empty in early-init contexts); -// AssigneeMatches refuses the close when the bead is assigned to someone else -// unless force is true. This is the authority guard for be-035. -func validateIssueClosable(id string, issue *types.Issue, actor string, force bool) error { - // Note: We use individual validators instead of ForClose() to maintain - // backward compatibility - the original didn't check for nil issues. - return validation.Chain( - validation.NotTemplate(), - validation.NotPinned(force), - validation.AssigneeMatches(actor, force), - )(id, issue) -} - // validateIssueReassignable checks whether an assignee update may proceed: // plain `bd update -a` / `bd assign` must not silently overwrite another // actor's live in_progress claim (bd-98s5c) — it was the last unfenced diff --git a/cmd/bd/show_unit_helpers_test.go b/cmd/bd/show_unit_helpers_test.go index 0d956e135..02dfd6bdf 100644 --- a/cmd/bd/show_unit_helpers_test.go +++ b/cmd/bd/show_unit_helpers_test.go @@ -22,65 +22,6 @@ func TestValidateIssueUpdatable(t *testing.T) { } } -func TestValidateIssueClosable(t *testing.T) { - if err := validateIssueClosable("x", nil, "alice", false); err != nil { - t.Fatalf("expected nil error, got %v", err) - } - if err := validateIssueClosable("bd-1", &types.Issue{IsTemplate: true}, "alice", false); err == nil { - t.Fatalf("expected template close error") - } - if err := validateIssueClosable("bd-2", &types.Issue{Status: types.StatusPinned}, "alice", false); err == nil { - t.Fatalf("expected pinned close error") - } - if err := validateIssueClosable("bd-2", &types.Issue{Status: types.StatusPinned}, "alice", true); err != nil { - t.Fatalf("expected pinned close to succeed with force, got %v", err) - } - - // ga-z3vht: pinned=true protects the bead independently of status, so - // `bd close` refuses it without --force on both the direct and proxied path. - booleanPinned := &types.Issue{Status: types.StatusOpen, Pinned: true} - if err := validateIssueClosable("bd-6", booleanPinned, "alice", false); err == nil { - t.Fatalf("expected boolean-pinned close error") - } - if err := validateIssueClosable("bd-6", booleanPinned, "alice", true); err != nil { - t.Fatalf("expected boolean-pinned close to succeed with force, got %v", err) - } - - // ga-ktn9pe.4.8: a closed row carrying pinned=true is the residue left by a - // forced close, and this guard still refuses it — closed status earns no - // exemption from the boolean trigger. Idempotent re-close was restored by - // ORDERING instead: cmd/bd/close.go and close_proxied_server.go skip close - // validation entirely for a row already closed at resolve time, so this guard - // is never reached on the no-op retry. Do not "simplify" it by exempting - // closed here — that would also disarm the guard on live status transitions. - closedPinnedResidue := &types.Issue{Status: types.StatusClosed, Pinned: true} - if err := validateIssueClosable("bd-7", closedPinnedResidue, "alice", false); err == nil { - t.Fatalf("expected closed+pinned residue to refuse a plain close") - } - if err := validateIssueClosable("bd-7", closedPinnedResidue, "alice", true); err != nil { - t.Fatalf("expected closed+pinned residue close to succeed with force, got %v", err) - } - - // be-035: actor != assignee must be refused without --force. - mismatched := &types.Issue{Assignee: "bob"} - if err := validateIssueClosable("bd-3", mismatched, "alice", false); err == nil { - t.Fatalf("expected actor/assignee mismatch error") - } - // --force overrides the authority check. - if err := validateIssueClosable("bd-3", mismatched, "alice", true); err != nil { - t.Fatalf("expected close to succeed with force despite mismatch, got %v", err) - } - // Same-actor close is allowed. - if err := validateIssueClosable("bd-4", &types.Issue{Assignee: "alice"}, "alice", false); err != nil { - t.Fatalf("expected matching-assignee close to succeed, got %v", err) - } - // Unassigned beads can be closed by anyone (lots of bd's flow involves - // closing beads nobody claimed). - if err := validateIssueClosable("bd-5", &types.Issue{Assignee: ""}, "alice", false); err != nil { - t.Fatalf("expected unassigned close to succeed, got %v", err) - } -} - func TestFindRepliesToAndReplies_WorksWithDoltStorage(t *testing.T) { ctx := context.Background() st := newTestStoreWithPrefix(t, filepath.Join(t.TempDir(), "test.db"), "test") diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index fab98a6f5..2cc93e90c 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -72,6 +72,7 @@ go_test( "brief_projection_test.go", "claim_next_test.go", "claim_test.go", + "close_guards_test.go", "close_test.go", "comments_test.go", "context_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index b09477c7c..1d12b9fa7 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -393,7 +393,7 @@ type ApplyCloseItem struct { // IT IS A STRING, and it must be the `revision` string a response carried, verbatim. A JSON number — or any other type — is a `400` naming this member. The token spans the FULL 64-bit range, so a number would be rounded past 2^53 by an IEEE-754-double parser and the guard would miss a row nothing else touched; a string round-trips exactly in every consumer. ExpectedVersion *string `json:"expected_version,omitempty"` - // Force Bypasses close policy — the open-children refusal and the live-blocker refusal — and nothing else. + // Force Bypasses close policy — the open-children refusal and the live-blocker refusal — and the pin and the assignee guard, and nothing else: never the template guard. // // CLOSE POLICY EVALUATES AT THIS ITEM, against the row as this request has already changed it. A LATER item that gives a closed parent an open child is NOT refused: the policy is a gate on the closing act, not an invariant the store maintains. Force *bool `json:"force,omitempty"` @@ -764,7 +764,7 @@ type BatchCloseRequest struct { // Actor Who is closing. `ClaimRequest.actor`'s rules exactly, and the value is recorded against every item. Actor string `json:"actor"` - // Force Bypass close policy — the open-children refusal and the live-blocker refusal — for EVERY item, and nothing else. It never bypasses validation and it never bypasses existence: an id that names nothing refuses whether or not this is set. It is request-wide because the flag that spells it is. + // Force Bypass close policy — the open-children refusal and the live-blocker refusal — and the pin and the assignee guard for EVERY item, and nothing else. It never bypasses the template guard or validation, and it never bypasses existence: an id that names nothing refuses whether or not this is set. It is request-wide because the flag that spells it is. Force *bool `json:"force,omitempty"` // Items The issues to close, in the order the caller asked for them. Every item appears in `outcomes` at the same index. @@ -929,7 +929,7 @@ type CloseIssueRequest struct { // IT IS A STRING, and it must be the `revision` string a response carried, verbatim. A JSON number — or any other type — is a `400` naming this member. The token spans the FULL 64-bit range, so a number would be rounded past 2^53 by an IEEE-754-double parser and the guard would miss a row nothing else touched; a string round-trips exactly in every consumer. ExpectedVersion *string `json:"expected_version,omitempty"` - // Force Bypass close policy — the open-children refusal and the live-blocker refusal — and nothing else. The refusals are the ROLE's, so this endpoint cannot skip a guard by forgetting one exists. A forced close still reports `open_children`. + // Force Bypass close policy — the open-children refusal and the live-blocker refusal — and the pin and the assignee guard, and nothing else: never the template guard (see "Close guards" on `POST /v0/beads/issues/{id}:close`). The refusals are the ROLE's, so this endpoint cannot skip a guard by forgetting one exists. A forced close still reports `open_children`. // // IT BYPASSES POLICY, NEVER A PRECONDITION. `expected_version` is still checked with it set, for the reason `issueops.CloseRequest.Force` gives: a caller saying "close it anyway" has said nothing about whether the row is still the one it read. Force *bool `json:"force,omitempty"` @@ -969,10 +969,13 @@ type CloseOutcome struct { // A BATCH WHOSE ITEMS ARE ALL `true` LANDED NOTHING, and records no history entry: a per-item success that changed nothing is not work the caller did. AlreadyClosed *bool `json:"already_closed,omitempty"` + // Assignee With `not_assignee`: the issue's assignee the refusing transaction observed, exactly as `Problem.assignee` carries it for the single close. Absent on a successful item and on every other refusal. + Assignee *string `json:"assignee,omitempty"` + // Blockers With `not_closable` on the live-blocker refusal: the blockers that refused THIS item, exactly as `Problem.blockers` carries them for the single close, and optional for the same reason. Absent on a successful item and on every other refusal. Blockers *[]Blocker `json:"blockers,omitempty"` - // Code This item's refusal, from `Problem.code`'s vocabulary and restricted to `not_found` (the id names no row in either plane) and `not_closable` (close policy refused it: open children, or a live blocker — see `open_children`). ABSENT means the item succeeded. + // Code This item's refusal, from `Problem.code`'s vocabulary and restricted to `not_found` (the id names no row in either plane), `not_closable` (close policy refused it: open children, or a live blocker — see `open_children`), and the close guards `template_read_only`, `issue_pinned` and `not_assignee` (see `assignee`), exactly as the single close answers them. ABSENT means the item succeeded. // // It is the problem vocabulary rather than a second one because an item refusal and a request refusal are the same question asked at two scopes, and a client that had to learn two vocabularies to classify one condition would be classifying the SCOPE rather than the condition. Code *string `json:"code,omitempty"` @@ -1586,7 +1589,7 @@ type Problem struct { // IT IS A STRING, and it must be the `revision` string a response carried, verbatim. A JSON number — or any other type — is a `400` naming this member. The token spans the FULL 64-bit range, so a number would be rounded past 2^53 by an IEEE-754-double parser and the guard would miss a row nothing else touched; a string round-trips exactly in every consumer. ActualVersion *string `json:"actual_version,omitempty"` - // Assignee With `already_claimed`: the actor currently holding the issue, read inside the transaction that refused. + // Assignee With `already_claimed`: the actor currently holding the issue, read inside the transaction that refused. With `not_assignee`: the issue's assignee the refusing close observed — always present on that code. // // IT IS OPTIONAL ON EVERY OPERATION BUT THE CLAIM. `POST /v0/beads/issues/{id}:claim` always carries it, because its conflict path reads the row it lost to. `PATCH /v0/beads/issues/{id}` and `POST /v0/beads/issues:batchApply` carry it only when the refusing transaction reported a holder, and `POST /v0/beads/issues/{id}:release` never does — the ownership fence refuses without naming anyone. An absent member means "this refusal could not name the holder", never "nobody holds it"; re-read the row. Assignee *string `json:"assignee,omitempty"` @@ -1607,7 +1610,7 @@ type Problem struct { // IT IS OPTIONAL. A refusal that could not name its blockers omits it and keeps the generic `detail`; absence means "this refusal did not name them", never "nothing blocks it". Re-read the issue's dependencies then. Blockers *[]Blocker `json:"blockers,omitempty"` - // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `template_read_only` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). + // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `template_read_only` (409), `issue_pinned` (409), `not_assignee` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). Code string `json:"code"` // DeclaredLater With `invalid_argument` on a batch operation whose items may name each other: whether the unresolvable key IS declared by the request, at a LATER index. diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index 8b2e3fcba..a1cfa73f0 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -1054,9 +1054,12 @@ func (s *Server) failApplyBatch(w http.ResponseWriter, r *http.Request, request s.fail(w, r, at(closeBlockedResult(err, "an item closes a blocked issue", "clear the blocker, or send the item's force flag"), "")) - // An update item naming a template: the shared mapping plus the item + // An update item naming a template, or the close guards on a close item: + // the shared mapping, carrying the holder for not_assignee, plus the item // members. - case errors.Is(err, issueops.ErrTemplateReadOnly): + case errors.Is(err, issueops.ErrTemplateReadOnly), + errors.Is(err, issueops.ErrPinned), + errors.As(err, new(*issueops.CloseNotAssigneeError)): s.fail(w, r, at(ClassifyError(err), "")) case errors.Is(err, storage.ErrAlreadyClaimed): diff --git a/internal/httpapi/batch_close.go b/internal/httpapi/batch_close.go index 9fb4cc02b..d28bea778 100644 --- a/internal/httpapi/batch_close.go +++ b/internal/httpapi/batch_close.go @@ -118,6 +118,7 @@ func closeOutcome(outcome issueops.CloseOutcome) apigen.CloseOutcome { } var openChildren *issueops.CloseOpenChildrenError + var notAssignee *issueops.CloseNotAssigneeError var ( code Code detail string @@ -135,6 +136,16 @@ func closeOutcome(outcome issueops.CloseOutcome) apigen.CloseOutcome { code = CodeNotClosable detail, wire.Blockers = closeBlocked(outcome.Err, "this issue is blocked", "clear the blocker, or send `force`") + // The close guards answer per item exactly as the single close answers + // them — the shared mapping's code and detail — with the holder on the + // outcome's own `assignee` member. + case errors.Is(outcome.Err, issueops.ErrTemplateReadOnly), + errors.Is(outcome.Err, issueops.ErrPinned), + errors.As(outcome.Err, ¬Assignee): + res := ClassifyError(outcome.Err) + code, detail = Code(res.Problem.Code), derefString(res.Problem.Detail) + wire.Assignee = res.Problem.Assignee + case errors.Is(outcome.Err, storage.ErrNotFound): code, detail = CodeNotFound, "no issue with this id in either plane" diff --git a/internal/httpapi/close_guards_test.go b/internal/httpapi/close_guards_test.go new file mode 100644 index 000000000..fb7a0e0ef --- /dev/null +++ b/internal/httpapi/close_guards_test.go @@ -0,0 +1,163 @@ +package httpapi + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// The close guards — template read-only, the pin, the assignee fence — are the +// ROLE's (issueops.Lifecycle.Close and the batch roles that share its body), so +// these cases drive a fake role that refuses the way the real one does and pin +// what the wire makes of it on each of the three close routes. The real role +// behind a real server is the served leg of the conformance contracts +// (internal/httpclient: TestServedLifecycleCloseEnforcesTheCloseGuards and its +// batch siblings), which also proves the client rebuilds the typed error. + +// closeGuardCases are the three refusals as a unit-of-work backend returns +// them: wrapped by the use case and the repository, so the mapping has to find +// them by type rather than by being handed the bare value. +var closeGuardCases = []struct { + name string + err error + code Code + assignee any +}{ + { + name: "template", + err: &issueops.TemplateReadOnlyError{IssueID: "bd-1"}, + code: CodeTemplateReadOnly, + }, + { + name: "pinned", + err: &issueops.PinnedError{IssueID: "bd-1"}, + code: CodeIssuePinned, + }, + { + name: "not assignee", + err: &issueops.CloseNotAssigneeError{IssueID: "bd-1", Assignee: "bob", Actor: "alice"}, + code: CodeNotAssignee, + assignee: "bob", + }, +} + +func wrapAsUOW(err error) error { + return fmt.Errorf("close bd-1: db: IssueSQLRepository.CloseChecked bd-1: %w", err) +} + +// assertNoRoleProse checks a detail is this server's own words: the role's +// sentence names the actor the caller sent and the holder, and a refusal's +// detail is not where a client reads either. +func assertNoRoleProse(t *testing.T, where string, detail any) { + t.Helper() + text, _ := detail.(string) + if text == "" { + t.Errorf("%s detail = %v, want this server's own sentence", where, detail) + } + for _, leaked := range []string{"cannot modify", "cannot close", "IssueSQLRepository"} { + if strings.Contains(text, leaked) { + t.Errorf("%s detail %q carries the role's prose (%q)", where, text, leaked) + } + } +} + +func TestCloseGuardsAreTypedConflicts(t *testing.T) { + for _, tc := range closeGuardCases { + t.Run(tc.name, func(t *testing.T) { + lifecycle := &roleLifecycle{closeErr: wrapAsUOW(tc.err)} + ts := newCloseServer(t, lifecycle) + + resp := ts.closeIssue(t, closePath, `{"actor":"alice"}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(tc.code) { + t.Errorf("code = %v, want %s", body["code"], tc.code) + } + if body["assignee"] != tc.assignee { + t.Errorf("assignee = %v, want %v", body["assignee"], tc.assignee) + } + if _, ok := body["open_children"]; ok { + t.Errorf("a close guard carries open_children, which would read as close policy") + } + assertNoRoleProse(t, "close", body["detail"]) + }) + } +} + +// TestCloseForwardsForceToTheGuards pins that the handler hands `force` to the +// role and decides nothing itself: whether a guard is waived is the role's call. +func TestCloseForwardsForceToTheGuards(t *testing.T) { + lifecycle := &roleLifecycle{closeErr: wrapAsUOW(&issueops.TemplateReadOnlyError{IssueID: "bd-1"})} + ts := newCloseServer(t, lifecycle) + + resp := ts.closeIssue(t, closePath, `{"actor":"alice","force":true}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want the role's 409 under force: %s", resp.StatusCode, readAll(t, resp)) + } + if got := lifecycle.closeRequests(); len(got) != 1 || !got[0].Force { + t.Fatalf("the role received %+v, want one request carrying Force", got) + } +} + +func TestBatchCloseItemsCarryTheCloseGuards(t *testing.T) { + outcomes := []issueops.CloseOutcome{{IssueID: "bd-0", Issue: closedIssue("bd-0"), Changed: true}} + for _, tc := range closeGuardCases { + outcomes = append(outcomes, issueops.CloseOutcome{IssueID: "bd-1", Err: wrapAsUOW(tc.err)}) + } + ts := newBatchCloseServer(t, &roleBatchCloser{result: issueops.CloseBatchResult{Outcomes: outcomes}}) + + resp := ts.batchClose(t, `{"actor":"alice","items":[{"id":"bd-0"},{"id":"bd-1"},{"id":"bd-1"},{"id":"bd-1"}]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 carrying per-item refusals: %s", resp.StatusCode, readAll(t, resp)) + } + got := outcomesOf(t, resp) + if len(got) != 4 { + t.Fatalf("outcomes = %d, want 4", len(got)) + } + if _, refused := got[0]["code"]; refused { + t.Errorf("outcome 0 = %v, want the plain item's success", got[0]) + } + for i, tc := range closeGuardCases { + item := got[i+1] + if item["code"] != string(tc.code) { + t.Errorf("%s item code = %v, want %s", tc.name, item["code"], tc.code) + } + if item["assignee"] != tc.assignee { + t.Errorf("%s item assignee = %v, want %v", tc.name, item["assignee"], tc.assignee) + } + if _, ok := item["issue"]; ok { + t.Errorf("%s item carries an issue snapshot beside its refusal", tc.name) + } + assertNoRoleProse(t, tc.name+" item", item["detail"]) + } +} + +func TestApplyBatchCloseItemGuardsAreTypedConflicts(t *testing.T) { + for _, tc := range closeGuardCases { + t.Run(tc.name, func(t *testing.T) { + applier := &roleBatchApplier{err: itemErr(1, issueops.ItemClose, "", "bd-1", wrapAsUOW(tc.err))} + ts := newApplyBatchServer(t, applier) + + resp := ts.claim(t, batchApplyPath, `{"actor":"alice","items":[{"kind":"create","create":{"title":"one"}}]}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(tc.code) { + t.Errorf("code = %v, want %s", body["code"], tc.code) + } + if body["assignee"] != tc.assignee { + t.Errorf("assignee = %v, want %v", body["assignee"], tc.assignee) + } + if body["item_index"] != float64(1) || body["item_kind"] != "close" || body["item_issue_id"] != "bd-1" { + t.Errorf("item members = %v/%v/%v, want 1/close/bd-1", body["item_index"], body["item_kind"], body["item_issue_id"]) + } + assertNoRoleProse(t, "applyBatch", body["detail"]) + }) + } +} diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index a7694e09a..2bad156a1 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -146,6 +146,23 @@ const ( // A 409 for CodeNotClosable's reason: the body is well-formed and the // STATE of the named row refuses it. CodeTemplateReadOnly Code = "template_read_only" + // CodeIssuePinned is an unforced close of a pinned issue, by either of the + // two spellings bd pins with (the `pinned` flag or the `pinned` status). + // `force` is the bypass. It is its own code rather than a third + // `not_closable` because that code's member-presence discriminator already + // reads "no `open_children`" as a live blocker, and a pin is not one. + CodeIssuePinned Code = "issue_pinned" + // CodeNotAssignee is an unforced close by an actor who is not the issue's + // assignee (be-035): an unassigned issue is closable by anyone, an + // assigned one by its holder, compared separator-insensitively. `force` is + // the bypass. The holder the refusing transaction observed travels in the + // `assignee` member, so a client names it without reading `detail`; the + // closing actor is the one the request sent. + // + // It is minted rather than folded into CodeAlreadyClaimed, which a client + // maps to a lost CLAIM: answering it for a close would have the client + // read the operation, rather than the code, to decide what was refused. + CodeNotAssignee Code = "not_assignee" // CodeDependencyCycle covers BOTH never-makes-progress refusals a requested // edge set can earn: a scheduling cycle, and a blocking edge against the // issue's own ancestor or descendant. They are one code because they have @@ -273,6 +290,8 @@ var codeStatus = map[Code]int{ CodeNotClosable: http.StatusConflict, CodeNotesOverwrite: http.StatusConflict, CodeTemplateReadOnly: http.StatusConflict, + CodeIssuePinned: http.StatusConflict, + CodeNotAssignee: http.StatusConflict, CodeNotReleasable: http.StatusConflict, CodeDependencyCycle: http.StatusConflict, CodeDependencyExists: http.StatusConflict, @@ -823,7 +842,8 @@ var operationCodes = map[string][]Code{ // The NARROWEST write vocabulary on this surface, and the narrowness is the // contract rather than an oversight. A problem document from this operation // means the batch never ran; every refusal an ITEM can earn — not_found for - // an id naming no row, not_closable for close policy — travels in that + // an id naming no row, not_closable for close policy, template_read_only, + // issue_pinned and not_assignee for the close guards — travels in that // item's outcome inside a 200. A 404 here would say the operation went to // the wrong place, and a 409 would say the whole batch was refused, and // neither is ever true of a per-item refusal. @@ -856,14 +876,18 @@ var operationCodes = map[string][]Code{ CodeAlreadyClaimed, CodeNotReleasable, CodePreconditionFailed, CodeBusy, CodeDBUnavailable, CodeInternal, }, - // TWO 409s, and they answer different questions with `code` as the only - // discriminator a client needs. + // Several 409s, and they answer different questions with `code` as the + // only discriminator a client needs. // // not_closable is close POLICY: an unforced close refused for open children - // or for a live blocker. There is no already_claimed here — closing work - // somebody else holds is not a refusal on this surface — and the idempotent - // re-close is a 200 carrying `already_closed`, the claim's answer to the - // same question. + // or for a live blocker. The idempotent re-close is a 200 carrying + // `already_closed`, the claim's answer to the same question. + // + // template_read_only, issue_pinned and not_assignee are the CLOSE GUARDS + // the role applies before policy, in that order: a template (no bypass), a + // pin and closing work somebody else holds (both bypassed by `force`). + // There is no already_claimed: that code is a lost claim, and not_assignee + // carries the holder in `assignee` itself. // // precondition_failed is `expected_version`'s, in the 409 form for the // reason that code documents, and it is CHECKED FIRST — before policy and @@ -872,6 +896,7 @@ var operationCodes = map[string][]Code{ // members make unrelated claims. OpCloseIssue: { CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, CodeNotClosable, CodePreconditionFailed, + CodeTemplateReadOnly, CodeIssuePinned, CodeNotAssignee, CodeBusy, CodeDBUnavailable, CodeInternal, }, // ONE 409, and it is a PRECONDITION rather than a policy. Close has a policy @@ -978,7 +1003,7 @@ var operationCodes = map[string][]Code{ OpApplyBatch: { CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, CodePreconditionFailed, CodeNotClosable, CodeAlreadyClaimed, CodeNotesOverwrite, CodeAlreadyExists, - CodeTemplateReadOnly, + CodeTemplateReadOnly, CodeIssuePinned, CodeNotAssignee, CodeDependencyCycle, CodeDependencyExists, CodeBusy, CodeDBUnavailable, CodeInternal, }, @@ -1525,6 +1550,7 @@ func ClassifyError(err error) Result { // `if` away from a pruned-past checkpoint arriving as a generic 500 on // whichever arm forgot it. var truncated *storage.EventsJournalTruncatedError + var notAssignee *issueops.CloseNotAssigneeError switch { case err == nil: @@ -1560,11 +1586,20 @@ func ClassifyError(err error) Result { case errors.Is(err, issueops.ErrCloseBlocked): return closeBlockedResult(err, "issue is blocked", "clear the blocker or close with force") - // The detail is this server's own words rather than the role's message, - // for the reason Problem.detail gives. + // The close guards; the first, template read-only, is every update's guard + // too. Each detail is this server's own words rather than the role's + // message, for the reason Problem.detail gives; the typed members are what + // a client reconstructs the refusal from. case errors.Is(err, issueops.ErrTemplateReadOnly): return newResult(CodeTemplateReadOnly, "this issue is a template, and templates are read-only; pour it to create work") + case errors.Is(err, issueops.ErrPinned): + return newResult(CodeIssuePinned, "this issue is pinned; close with force to override") + + case errors.As(err, ¬Assignee): + return newResult(CodeNotAssignee, + "this issue is assigned to another actor; reclaim it, or close with force").WithAssignee(notAssignee.Assignee) + case errors.Is(err, ErrBusy): res := newResult(CodeBusy, "") res.RetryAfterSeconds = retryAfterSaturation diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 6d9c5b354..f62181bf3 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -2420,8 +2420,10 @@ paths: `closed_by_session` and `close_reason` — written under first-close-wins by `{id}:close`, which a patch write would bypass. A `status` that crosses into the done category still answers to close POLICY here; it - does not acquire the close's semantics, which is why `{id}:close` - remains the operation to reach for when what you mean is "close this". + does not acquire the close's semantics, and that includes its close + guards: a patch is never refused `issue_pinned` or `not_assignee`. + That is why `{id}:close` remains the operation to reach for when what + you mean is "close this". `created_at`, `created_by` — `POST /v0/beads/issues` withholds them for @@ -3202,8 +3204,33 @@ paths: the blockers themselves travel in `blockers`). The MEMBER'S PRESENCE is the discriminator, so telling the two apart never requires reading `detail`. Both refusals are the ROLE's, so `force` - bypasses those two and nothing else — this endpoint cannot skip a guard - by forgetting one exists. + bypasses those two — this endpoint cannot skip a guard by forgetting + one exists. + + + ## Close guards + + + Before close policy, a close of an issue that is not already closed + answers to three guards, in this order. A template refuses with + `409` / `template_read_only` and has no bypass. A pinned issue (the + `pinned` flag or the `pinned` status) refuses with `409` / + `issue_pinned`. An issue assigned to someone other than `actor` + (compared separator-insensitively; an unassigned issue is closable by + anyone) refuses with `409` / `not_assignee`, carrying the holder in + `assignee`. `force` bypasses the pin and the assignee guard. They are + the guards `bd close` applies, and they are the ROLE's, so every close + operation — this one, `POST /v0/beads/issues:batchClose` and a + `close` item of `POST /v0/beads/issues:batchApply` — applies them + alike. An idempotent re-close is not guarded: it changes nothing. + + + The guards gate the close operations, not the closed state. A status + update that crosses into the done category — a `PATCH` of + `/v0/beads/issues/{id}` or an `update` item of + `POST /v0/beads/issues:batchApply` — is not a close operation and + answers to close policy only, so it still closes a pinned issue or one + another actor holds. ## The guard @@ -3297,8 +3324,13 @@ paths: bypass both. `force` does NOT bypass the guard above. - Nothing was written in either case. - x-bd-codes: [not_closable, precondition_failed] + `template_read_only`, `issue_pinned` and `not_assignee`: a close + guard refused (see "Close guards"). `not_assignee` carries the + holder in `assignee`. `force` bypasses the latter two. + + + Nothing was written in any case. + x-bd-codes: [issue_pinned, not_assignee, not_closable, precondition_failed, template_read_only] content: application/problem+json: schema: @@ -4338,9 +4370,16 @@ paths: per-item bypasses, and a forced item never earns this. - `template_read_only` is an `update` item whose target is a - template, exactly as `PATCH /v0/beads/issues/{id}` answers it; no - item member bypasses it. + `template_read_only` is an `update` or a `close` item whose target + is a template, exactly as `PATCH /v0/beads/issues/{id}` and + `POST /v0/beads/issues/{id}:close` answer it; no item member + bypasses it. + + + `issue_pinned` and `not_assignee` are the other two close guards + refusing a `close` item, exactly as + `POST /v0/beads/issues/{id}:close` answers them; `not_assignee` + carries the holder in `assignee`. `close.force` bypasses both. `already_claimed` is the assignee fence: an `update` whose @@ -4372,8 +4411,8 @@ paths: a malformed request. x-bd-codes: [already_claimed, already_exists, dependency_cycle, - dependency_exists, not_closable, notes_overwrite_refused, precondition_failed, - template_read_only] + dependency_exists, issue_pinned, not_assignee, not_closable, + notes_overwrite_refused, precondition_failed, template_read_only] content: application/problem+json: schema: @@ -4421,8 +4460,9 @@ paths: contract: a refusal OF THE REQUEST is a problem document, and a refusal OF AN ITEM is a member of that item's outcome. `code` on an outcome is the same vocabulary `Problem.code` publishes, restricted to - `not_found` and `not_closable`, so a client classifies an item exactly - as it classifies a request. + `not_found`, `not_closable` and the three close guards + (`template_read_only`, `issue_pinned`, `not_assignee`), so a client + classifies an item exactly as it classifies a request. WHAT IS ATOMIC IS EVERYTHING THAT LANDS: one transaction, at most one @@ -4466,8 +4506,9 @@ paths: NO PER-ITEM PRECONDITION. There is no counterpart here to a compare-and-set guard, matching the role, so `force` is a question with - only two answers: it bypasses blocker and open-child close policy for - every item, and it never bypasses validation or existence. + only two answers: it bypasses blocker and open-child close policy, the + pin and the assignee guard for every item, and it never bypasses the + template guard, validation or existence. ## Hooks and auto-commit @@ -10452,7 +10493,8 @@ components: default: false description: >- Bypasses close policy — the open-children refusal and the - live-blocker refusal — and nothing else. + live-blocker refusal — and the pin and the assignee guard, and + nothing else: never the template guard. CLOSE POLICY EVALUATES AT THIS ITEM, against the row as this request @@ -11065,10 +11107,11 @@ components: default: false description: >- Bypass close policy — the open-children refusal and the live-blocker - refusal — for EVERY item, and nothing else. It never bypasses - validation and it never bypasses existence: an id that names nothing - refuses whether or not this is set. It is request-wide because the - flag that spells it is. + refusal — and the pin and the assignee guard for EVERY item, and + nothing else. It never bypasses the template guard or validation, + and it never bypasses existence: an id that names nothing refuses + whether or not this is set. It is request-wide because the flag + that spells it is. BatchCloseItem: type: object @@ -11161,13 +11204,22 @@ components: refused THIS item, exactly as `Problem.blockers` carries them for the single close, and optional for the same reason. Absent on a successful item and on every other refusal. + assignee: + type: string + description: >- + With `not_assignee`: the issue's assignee the refusing transaction + observed, exactly as `Problem.assignee` carries it for the single + close. Absent on a successful item and on every other refusal. code: type: string description: >- This item's refusal, from `Problem.code`'s vocabulary and restricted - to `not_found` (the id names no row in either plane) and + to `not_found` (the id names no row in either plane), `not_closable` (close policy refused it: open children, or a live - blocker — see `open_children`). ABSENT means the item succeeded. + blocker — see `open_children`), and the close guards + `template_read_only`, `issue_pinned` and `not_assignee` (see + `assignee`), exactly as the single close answers them. ABSENT means + the item succeeded. It is the problem vocabulary rather than a second one because an @@ -11246,9 +11298,11 @@ components: default: false description: >- Bypass close policy — the open-children refusal and the - live-blocker refusal — and nothing else. The refusals are the - ROLE's, so this endpoint cannot skip a guard by forgetting one - exists. A forced close still reports `open_children`. + live-blocker refusal — and the pin and the assignee guard, and + nothing else: never the template guard (see "Close guards" on + `POST /v0/beads/issues/{id}:close`). The refusals are the ROLE's, + so this endpoint cannot skip a guard by forgetting one exists. A + forced close still reports `open_children`. IT BYPASSES POLICY, NEVER A PRECONDITION. `expected_version` is @@ -12185,7 +12239,8 @@ components: configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), - `template_read_only` (409), + `template_read_only` (409), `issue_pinned` (409), + `not_assignee` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), @@ -12245,7 +12300,9 @@ components: type: string description: >- With `already_claimed`: the actor currently holding the issue, read - inside the transaction that refused. + inside the transaction that refused. With `not_assignee`: the + issue's assignee the refusing close observed — always present on + that code. IT IS OPTIONAL ON EVERY OPERATION BUT THE CLAIM. `POST diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 89c866bab..02a66f58c 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -1220,6 +1220,12 @@ "type": "boolean", "required": false }, + { + "schema": "CloseOutcome", + "member": "assignee", + "type": "string", + "required": false + }, { "schema": "CloseOutcome", "member": "blockers", diff --git a/internal/httpclient/batchcloser.go b/internal/httpclient/batchcloser.go index a9d231e3c..d7257186a 100644 --- a/internal/httpclient/batchcloser.go +++ b/internal/httpclient/batchcloser.go @@ -239,7 +239,7 @@ func decodeBatchCloseResult(req issueops.CloseBatchRequest, resp *apigen.BatchCl } outcomes := make([]issueops.CloseOutcome, len(resp.Outcomes)) for i := range resp.Outcomes { - outcomes[i] = decodeBatchCloseOutcome(resp.Outcomes[i]) + outcomes[i] = decodeBatchCloseOutcome(req.Actor, resp.Outcomes[i]) } return issueops.CloseBatchResult{Outcomes: outcomes}, nil } @@ -252,10 +252,10 @@ func decodeBatchCloseResult(req issueops.CloseBatchRequest, resp *apigen.BatchCl // members are FLAT on the outcome rather than nested under an error object, so // `open_children` is read from the same field in both branches and means two // different things depending on which one — see the schema. -func decodeBatchCloseOutcome(item apigen.CloseOutcome) issueops.CloseOutcome { +func decodeBatchCloseOutcome(actor string, item apigen.CloseOutcome) issueops.CloseOutcome { out := issueops.CloseOutcome{IssueID: item.IssueId} if item.Code != nil { - out.Err = perItemCloseError(item.IssueId, item) + out.Err = perItemCloseError(item.IssueId, actor, item) return out } // A successful outcome carries the snapshot; already_closed is the idempotent @@ -276,6 +276,10 @@ func decodeBatchCloseOutcome(item apigen.CloseOutcome) issueops.CloseOutcome { // not_found -> issueops.ErrNotFound // not_closable + open_children -> *issueops.CloseOpenChildrenError // not_closable -> issueops.ErrCloseBlocked +// template_read_only -> *issueops.TemplateReadOnlyError +// issue_pinned -> *issueops.PinnedError +// not_assignee + assignee -> *issueops.CloseNotAssigneeError (the +// closing actor is the request's own) // an unknown code -> *wire.UnknownItemCodeError (the generic // typed per-item error, so version skew lands // as a typed refusal rather than a silent gap) @@ -285,7 +289,7 @@ func decodeBatchCloseOutcome(item apigen.CloseOutcome) issueops.CloseOutcome { // // It is only ever called with a refused outcome — Code non-nil — which the one // caller above guarantees. -func perItemCloseError(issueID string, item apigen.CloseOutcome) error { +func perItemCloseError(issueID, actor string, item apigen.CloseOutcome) error { var code string if item.Code != nil { code = *item.Code @@ -298,6 +302,16 @@ func perItemCloseError(issueID string, item apigen.CloseOutcome) error { return &issueops.CloseOpenChildrenError{IssueID: issueID, OpenChildren: *item.OpenChildren} } return issueops.ErrCloseBlocked + case codeTemplateReadOnly: + return &issueops.TemplateReadOnlyError{IssueID: issueID} + case codeIssuePinned: + return &issueops.PinnedError{IssueID: issueID} + case codeNotAssignee: + var assignee string + if item.Assignee != nil { + assignee = stripItemControlRunes(*item.Assignee) + } + return &issueops.CloseNotAssigneeError{IssueID: issueID, Assignee: assignee, Actor: actor} default: var detail string if item.Detail != nil { @@ -311,12 +325,15 @@ func perItemCloseError(issueID string, item apigen.CloseOutcome) error { } } -// The two per-item refusal codes this outcome can carry. They are the +// The per-item refusal codes this outcome can carry. They are the // `Problem.code` spellings, restated here rather than imported because // importing the server package would drag the storage engine into a client. const ( - codeNotFound = "not_found" - codeNotClosable = "not_closable" + codeNotFound = "not_found" + codeNotClosable = "not_closable" + codeTemplateReadOnly = "template_read_only" + codeIssuePinned = "issue_pinned" + codeNotAssignee = "not_assignee" ) // stripItemControlRunes removes control runes from a server-controlled per-item @@ -363,6 +380,9 @@ func perItemCloseRefusal(issueID string, err error) (issueops.CloseOutcome, bool case errors.As(err, &openChildren): case errors.Is(err, issueops.ErrNotFound): case errors.Is(err, issueops.ErrCloseBlocked): + case errors.Is(err, issueops.ErrTemplateReadOnly): + case errors.Is(err, issueops.ErrPinned): + case errors.As(err, new(*issueops.CloseNotAssigneeError)): default: return issueops.CloseOutcome{}, false } diff --git a/internal/httpclient/lifecycle.go b/internal/httpclient/lifecycle.go index 1bad6d62c..409be20d3 100644 --- a/internal/httpclient/lifecycle.go +++ b/internal/httpclient/lifecycle.go @@ -936,11 +936,17 @@ func setNullableTime(out map[string]any, member string, field issueops.Field[*ti // operation the Lifecycle role uses (design D8's off-role list, extended by the // `bd close` decision). // -// Its one caller at tip is the molecule auto-close, which closes a parent whose -// children have all finished and discards nothing about the outcome. Routing it -// onto the role's operation rather than refusing it is what keeps that path -// working over http; it carries no reason-per-item and no force, so the mapping -// is total. +// Routing it onto the role's operation rather than refusing it is what keeps +// the molecule auto-close (which closes a parent whose children have all +// finished) and the other raw closes cmd/bd makes outside `bd close` working +// over http. It sends force because the raw close is the unguarded one +// everywhere else: dolt, embedded and proxied run it as +// issueops.CloseIssueInTx, which applies neither the close guards nor close +// policy, and its signature carries no force a caller could use to waive a +// refusal those backends never make. Sent unforced, a pinned or claimed root +// would stop closing over http and nowhere else. The template guard has no +// bypass on the wire, so a template still refuses here where the other +// backends close it; no raw caller is known to close one. func (s *Store) CloseIssue(ctx context.Context, id, reason, actor, session string) error { w, err := s.roleWire("CloseIssue") if err != nil { @@ -952,6 +958,6 @@ func (s *Store) CloseIssue(ctx context.Context, id, reason, actor, session strin if err := requireID("issue id", id); err != nil { return err } - _, err = w.CloseIssue(ctx, id, closeBody(actor, reason, session, false)) + _, err = w.CloseIssue(ctx, id, closeBody(actor, reason, session, true)) return err } diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index fa21e6b0c..1117c51b8 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -261,6 +261,10 @@ func TestServedBatchApplySplicesTheMetadataOfATemplateItCreates(t *testing.T) { conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, t.Context(), newServedBatchApplyFixture(t, "hbats")) } +func TestServedBatchApplyCloseItemsAnswerToTheCloseGuards(t *testing.T) { + conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, t.Context(), newServedBatchApplyFixture(t, "hbacg")) +} + // TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed wires the dep_add // HasSpawner/ThreadID lineage conformance case onto the http leg (S5): a // served default Config always advertises CapBatchApplyDepAddLineage (it is diff --git a/internal/httpclient/served_batch_close_test.go b/internal/httpclient/served_batch_close_test.go index 11b9a5145..9c3911ef4 100644 --- a/internal/httpclient/served_batch_close_test.go +++ b/internal/httpclient/served_batch_close_test.go @@ -168,3 +168,10 @@ func TestServedBatchCloserDoesNotMutateTheCallerRequest(t *testing.T) { func TestServedBatchCloserSettlesTheDependersOfWhatItClosed(t *testing.T) { conformance.RunBatchCloserSettlesTheDependersOfWhatItClosed(t, t.Context(), newServedBatchCloserFixture(t, "hbcbs")) } + +// TestServedBatchCloserItemsAnswerToTheCloseGuards is the served leg of the +// close guards: each guarded item travels back as its outcome's code (and +// `assignee`) and is rebuilt into the same typed error a local batch returns. +func TestServedBatchCloserItemsAnswerToTheCloseGuards(t *testing.T) { + conformance.RunBatchCloserItemsAnswerToTheCloseGuards(t, t.Context(), newServedBatchCloserFixture(t, "hbcgd")) +} diff --git a/internal/httpclient/served_claim_lifecycle_test.go b/internal/httpclient/served_claim_lifecycle_test.go index 9f8033046..08e5bc810 100644 --- a/internal/httpclient/served_claim_lifecycle_test.go +++ b/internal/httpclient/served_claim_lifecycle_test.go @@ -309,6 +309,75 @@ func TestServedLifecycleReopenReblocksItsDependers(t *testing.T) { conformance.RunLifecycleReopenReblocksItsDependers(t, t.Context(), newServedCloseReopenFixture(t, "hlbr")) } +// TestServedLifecycleCloseEnforcesTheCloseGuards is the served leg of the close +// guards: bd serve refuses with template_read_only, issue_pinned and +// not_assignee, and the client rebuilds the typed error — sentence included — +// the embedded store returns. +func TestServedLifecycleCloseEnforcesTheCloseGuards(t *testing.T) { + conformance.RunLifecycleCloseEnforcesTheCloseGuards(t, t.Context(), newServedCloseReopenFixture(t, "hlcg")) +} + +// TestServedRawCloseKeepsParityWithTheOtherBackends is the off-role half of the +// guards: Store.CloseIssue must close what the raw close closes on dolt, +// embedded and proxied, where issueops.CloseIssueInTx runs neither the guards +// nor close policy. A pin of either spelling, another actor's claim and an open +// child each refuse the Lifecycle close above; the raw close sends force, so a +// molecule auto-close of such a root still lands over http. The template is the +// one refusal left, because read-only has no bypass on the wire. +func TestServedRawCloseKeepsParityWithTheOtherBackends(t *testing.T) { + ctx := t.Context() + env := newServedEnv(t, "hrc") + + const ( + flagPinned = "hrc-flag-pinned" + statusPinned = "hrc-status-pinned" + held = "hrc-held" + parent = "hrc-parent" + child = "hrc-child" + template = "hrc-template" + ) + for _, issue := range []*types.Issue{ + {ID: flagPinned, Status: types.StatusOpen, Pinned: true}, + {ID: statusPinned, Status: types.StatusPinned}, + {ID: held, Status: types.StatusInProgress, Assignee: "holder"}, + {ID: parent, Status: types.StatusOpen}, + {ID: child, Status: types.StatusOpen}, + {ID: template, Status: types.StatusOpen, IsTemplate: true}, + } { + issue.Title, issue.Priority, issue.IssueType = issue.ID, 2, types.TypeTask + if err := env.createIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + if err := env.addDependency(ctx, &types.Dependency{IssueID: child, DependsOnID: parent, Type: types.DepParentChild}, "seed"); err != nil { + t.Fatalf("seed %s as a child of %s: %v", child, parent, err) + } + + assertStatus := func(id string, want types.Status) { + t.Helper() + got, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read back %s: %v", id, err) + } + if got.Status != want { + t.Errorf("%s has status %q after the raw close, want %q", id, got.Status, want) + } + } + + for _, id := range []string{flagPinned, statusPinned, held, parent} { + if err := env.subject.CloseIssue(ctx, id, "all steps complete", "closer", ""); err != nil { + t.Fatalf("raw close of %s over http: %v, want it to land as the raw close does on every other backend", id, err) + } + assertStatus(id, types.StatusClosed) + } + + err := env.subject.CloseIssue(ctx, template, "all steps complete", "closer", "") + if !errors.As(err, new(*issueops.TemplateReadOnlyError)) { + t.Fatalf("raw close of template %s over http: err = %v, want *TemplateReadOnlyError", template, err) + } + assertStatus(template, types.StatusOpen) +} + func TestServedLifecycleReopenLeavesNonDoneStatusesUnchanged(t *testing.T) { conformance.RunLifecycleReopenLeavesNonDoneStatusesUnchanged(t, t.Context(), newServedCloseReopenFixture(t, "hlrn")) } diff --git a/internal/httpclient/wire/client.go b/internal/httpclient/wire/client.go index 2c6fd6263..76955f51f 100644 --- a/internal/httpclient/wire/client.go +++ b/internal/httpclient/wire/client.go @@ -208,6 +208,10 @@ type Request struct { IssueID string DependsOnID string + // Actor is the acting identity the body carries, for a refusal whose typed + // error names it — the close's not_assignee — when the wire does not + // repeat what the request already said. + Actor string } // Do issues r and decodes a 2xx body into out, which may be nil for a response @@ -292,7 +296,7 @@ func (c *Client) Do(ctx context.Context, r Request, out any) error { } func (c *Client) target(r Request) target { - return target{op: r.Op, serverURL: c.base.Redacted(), issueID: r.IssueID, dependsOnID: r.DependsOnID, expectID: c.expectID} + return target{op: r.Op, serverURL: c.base.Redacted(), issueID: r.IssueID, dependsOnID: r.DependsOnID, actor: r.Actor, expectID: c.expectID} } // credentialSourceReporter is the optional half of a CredentialProvider: a diff --git a/internal/httpclient/wire/problem.go b/internal/httpclient/wire/problem.go index a02dc3d34..106cfea73 100644 --- a/internal/httpclient/wire/problem.go +++ b/internal/httpclient/wire/problem.go @@ -346,6 +346,9 @@ type target struct { serverURL string issueID string dependsOnID string + // actor is the request's acting identity (Request.Actor), which the + // not_assignee refusal names and the wire does not repeat. + actor string // expectID is the workspace's own pinned project id (Client.expectID), which // the wire never carries — it is a client-side fact. A project_mismatch // refusal names the server's own id in server_project_id; pairing it with this @@ -638,12 +641,20 @@ var codeSentinel = map[string]func(*ProblemError, target) error{ // sentinel callers already switch on locally.] "notes_overwrite_refused": func(*ProblemError, target) error { return issueops.ErrNotesOverwrite }, - // The template guard, rebuilt whole so a caller reads the same typed error - // — and the same message — the embedded store returns. The row comes from - // the request or, on a batch, from `item_issue_id`. + // The close guards (the first, template read-only, is every update's guard + // too), rebuilt whole so a caller reads the same typed error — and the same + // message — the embedded store returns. The row comes from the request or, + // on a batch, from `item_issue_id`; the holder from `assignee`; the closing + // actor from the request, which the wire does not repeat. "template_read_only": func(e *ProblemError, t target) error { return &issueops.TemplateReadOnlyError{IssueID: refusedIssueID(e, t)} }, + "issue_pinned": func(e *ProblemError, t target) error { + return &issueops.PinnedError{IssueID: refusedIssueID(e, t)} + }, + "not_assignee": func(e *ProblemError, t target) error { + return &issueops.CloseNotAssigneeError{IssueID: refusedIssueID(e, t), Assignee: deref(e.Assignee), Actor: t.actor} + }, // The same rule again: `issue_id` present means the edge named the issue's // own ancestor or descendant, absent means a plain scheduling cycle. The diff --git a/internal/httpclient/wire/writes.go b/internal/httpclient/wire/writes.go index c9557f1b1..c367198b1 100644 --- a/internal/httpclient/wire/writes.go +++ b/internal/httpclient/wire/writes.go @@ -160,7 +160,7 @@ func (c *Client) CloseIssue(ctx context.Context, id string, body apigen.CloseIss return nil, err } var out apigen.CloseIssueResponse - r := Request{Op: OpCloseIssue, Method: http.MethodPost, Path: path, Body: body, IssueID: id} + r := Request{Op: OpCloseIssue, Method: http.MethodPost, Path: path, Body: body, IssueID: id, Actor: body.Actor} if err := c.dispatch(ctx, r, &out); err != nil { return nil, err } @@ -539,7 +539,7 @@ func (c *Client) ApplyBatch(ctx context.Context, body ApplyBatchRequest) (*apige } var out apigen.ApplyBatchResponse - r := Request{Op: OpApplyBatch, Method: http.MethodPost, Path: PathIssuesBatchApply, Body: body} + r := Request{Op: OpApplyBatch, Method: http.MethodPost, Path: PathIssuesBatchApply, Body: body, Actor: body.Actor} if err := c.dispatch(ctx, r, &out); err != nil { return nil, err } diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index 480150703..8ead3a15b 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -265,6 +265,11 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) }) + t.Run("CloseItemsAnswerToTheCloseGuards", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) + }) } // newDoltBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/dolt/batch_closer_contract_test.go b/internal/storage/dolt/batch_closer_contract_test.go index 82337b90c..360ef1347 100644 --- a/internal/storage/dolt/batch_closer_contract_test.go +++ b/internal/storage/dolt/batch_closer_contract_test.go @@ -131,6 +131,12 @@ func TestBatchCloserSettlesTheDependersOfWhatItClosed(t *testing.T) { conformance.RunBatchCloserSettlesTheDependersOfWhatItClosed(t, ctx, fixture) } +func TestBatchCloserItemsAnswerToTheCloseGuards(t *testing.T) { + fixture, ctx, cleanup := newDoltBatchCloserFixture(t, "bcguard") + defer cleanup() + conformance.RunBatchCloserItemsAnswerToTheCloseGuards(t, ctx, fixture) +} + func newDoltBatchCloserFixture(t *testing.T, prefix string) (conformance.BatchCloserFixture, context.Context, func()) { t.Helper() store, storeCleanup := setupTestStore(t) diff --git a/internal/storage/dolt/issues.go b/internal/storage/dolt/issues.go index 3efcde47c..b6a8362b6 100644 --- a/internal/storage/dolt/issues.go +++ b/internal/storage/dolt/issues.go @@ -606,8 +606,10 @@ func (s *DoltStore) closeIssue(ctx context.Context, id string, reason string, ac // when it has a live direct blocker unless opts.Force is set, and — when // opts.ExpectedVersion is non-nil — with storage.ErrVersionMismatch when the // row's current RowVersion no longer matches (an orthogonal CAS that Force does -// not bypass). Both checks and the close share one transaction, so they are -// atomic (no TOCTOU). Mirrors CloseIssue's Dolt-specific concerns (wisp routing, +// not bypass). The close guards (template, pin, assignee; see +// issueops.CloseIssueCheckedInTx) refuse first, the template whatever Force +// says. Every check and the close share one transaction, so they are atomic +// (no TOCTOU). Mirrors CloseIssue's Dolt-specific concerns (wisp routing, // DOLT_ADD/COMMIT). func (s *DoltStore) CloseIssueChecked(ctx context.Context, id string, actor string, opts storage.CloseIssueOptions) (storage.CloseIssueResult, error) { var result storage.CloseIssueResult diff --git a/internal/storage/dolt/lifecycle_close_reopen_contract_test.go b/internal/storage/dolt/lifecycle_close_reopen_contract_test.go index 5fab44fa9..236909c36 100644 --- a/internal/storage/dolt/lifecycle_close_reopen_contract_test.go +++ b/internal/storage/dolt/lifecycle_close_reopen_contract_test.go @@ -80,6 +80,9 @@ func TestLifecycleCloseReopenContract(t *testing.T) { t.Run("ReopenReblocksItsDependers", func(t *testing.T) { conformance.RunLifecycleReopenReblocksItsDependers(t, ctx, fixture) }) + t.Run("CloseEnforcesTheCloseGuards", func(t *testing.T) { + conformance.RunLifecycleCloseEnforcesTheCloseGuards(t, ctx, fixture) + }) } func newDoltLifecycleCloseReopenFixture(t *testing.T, prefix string) (conformance.LifecycleCloseReopenFixture, context.Context, func()) { diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 4a1282e25..8c78e1ff5 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -223,9 +223,11 @@ func (r *issueSQLRepositoryImpl) Update(ctx context.Context, id string, updates // Close-policy parity with issueops.updateIssueInTx: a status that crosses // into the done category is a close by another name and answers to close - // policy. A refusal returns before any write and aborts the caller's unit of - // work. The wrap keeps the sentinels matchable, so a caller distinguishes - // these refusals here exactly as it does on the close path. + // policy, and, as there, to close policy only: the pin and assignee close + // guards stay on the close verbs (see updateIssueInTx for why). A refusal + // returns before any write and aborts the caller's unit of work. The wrap + // keeps the sentinels matchable, so a caller distinguishes these refusals + // here exactly as it does on the close path. if statusChanging { crossing, err := issueops.CrossesIntoDoneCategoryInTx(ctx, r.runner, oldIssue.Status, updates) if err != nil { diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index 5bf1afbaa..9280f9acb 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -193,6 +193,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("SplicesTheMetadataOfATemplateItCreates", func(t *testing.T) { conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) }) + t.Run("CloseItemsAnswerToTheCloseGuards", func(t *testing.T) { + conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) + }) } // newEmbeddedBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/embeddeddolt/batch_closer_contract_test.go b/internal/storage/embeddeddolt/batch_closer_contract_test.go index bb18d6b1d..8131d5fc8 100644 --- a/internal/storage/embeddeddolt/batch_closer_contract_test.go +++ b/internal/storage/embeddeddolt/batch_closer_contract_test.go @@ -133,6 +133,12 @@ func TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed(t *testing.T) { conformance.RunBatchCloserSettlesTheDependersOfWhatItClosed(t, ctx, newEmbeddedBatchCloserFixture(t, "bcblocked")) } +func TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunBatchCloserItemsAnswerToTheCloseGuards(t, ctx, newEmbeddedBatchCloserFixture(t, "bcguard")) +} + func newEmbeddedBatchCloserFixture(t *testing.T, prefix string) conformance.BatchCloserFixture { t.Helper() te := newTestEnv(t, prefix) diff --git a/internal/storage/embeddeddolt/issues.go b/internal/storage/embeddeddolt/issues.go index 7e1140204..bd67df66c 100644 --- a/internal/storage/embeddeddolt/issues.go +++ b/internal/storage/embeddeddolt/issues.go @@ -165,8 +165,10 @@ func (s *EmbeddedDoltStore) CloseIssue(ctx context.Context, id string, reason st // when it has a live direct blocker unless opts.Force is set, and — when // opts.ExpectedVersion is non-nil — with storage.ErrVersionMismatch when the // row's current RowVersion no longer matches (an orthogonal CAS that Force does -// not bypass). Both checks and the close share one transaction, so they are -// atomic (no TOCTOU). Delegates SQL work to issueops; EmbeddedDolt auto-commits +// not bypass). The close guards (template, pin, assignee; see +// issueops.CloseIssueCheckedInTx) refuse first, the template whatever Force +// says. Every check and the close share one transaction, so they are atomic +// (no TOCTOU). Delegates SQL work to issueops; EmbeddedDolt auto-commits // the transaction. func (s *EmbeddedDoltStore) CloseIssueChecked(ctx context.Context, id string, actor string, opts storage.CloseIssueOptions) (storage.CloseIssueResult, error) { var result storage.CloseIssueResult diff --git a/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go b/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go index 153b33232..c8fef9004 100644 --- a/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go +++ b/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go @@ -83,6 +83,9 @@ func TestLifecycleCloseReopenContract(t *testing.T) { t.Run("ReopenReblocksItsDependers", func(t *testing.T) { conformance.RunLifecycleReopenReblocksItsDependers(t, ctx, fixture) }) + t.Run("CloseEnforcesTheCloseGuards", func(t *testing.T) { + conformance.RunLifecycleCloseEnforcesTheCloseGuards(t, ctx, fixture) + }) } func newEmbeddedLifecycleCloseReopenFixture(t *testing.T, te *testEnv, prefix string) conformance.LifecycleCloseReopenFixture { diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index be4703360..3d7ab0898 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -22,6 +22,7 @@ go_library( "claim_next.go", "close.go", "close_batch.go", + "close_guard.go", "commenter.go", "comments.go", "commit_pending.go", @@ -136,6 +137,7 @@ go_test( "blocked_recheck_test.go", "blocked_state_result_test.go", "blocked_state_template_test.go", + "close_guard_test.go", "close_test.go", "config_helpers_test.go", "config_prefix_test.go", diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 188c704c1..03f9fbb94 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -54,6 +54,12 @@ func CloseIssueWithoutEventInTx(ctx context.Context, tx DBTX, id string, reason, // is_blocked column whose direct blockers have since closed. Reading the live // blocker list self-heals against a stale column instead of acting on it. // +// Before close policy it applies the close guards (enforceCloseGuardsInTx) to a +// target that is not already closed: a template refuses with +// *TemplateReadOnlyError whatever force says, and unless force is set a pinned +// issue refuses with *PinnedError and one assigned to someone other than actor +// with *CloseNotAssigneeError. +// // When expectedVersion is non-nil it adds an ORTHOGONAL optimistic-concurrency // precondition: the row's current RowVersion (row_lock) must still equal // *expectedVersion or the close refuses with storage.ErrVersionMismatch. This @@ -80,6 +86,14 @@ func CloseIssueCheckedInTx(ctx context.Context, tx DBTX, id, reason, actor, sess if !found { return nil, fmt.Errorf("%w: issue %s", storage.ErrNotFound, id) } + // The close guards run before close policy, so a target that trips both + // reads the guard's refusal, as `bd close` always printed it. They only + // read, so they need no savepoint. + if !closed { + if err := enforceCloseGuardsInTx(ctx, tx, id, targetColumn, actor, force); err != nil { + return nil, err + } + } // domain/db also supplies a *sql.DB Runner, whose independently pooled // statements cannot retain a savepoint. The shared UOW uses a pinned // *sql.Conn after START TRANSACTION, while embedded callers use *sql.Tx. diff --git a/internal/storage/issueops/close_guard.go b/internal/storage/issueops/close_guard.go new file mode 100644 index 000000000..9c715d89f --- /dev/null +++ b/internal/storage/issueops/close_guard.go @@ -0,0 +1,112 @@ +package issueops + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// closeGuardRow is the slice of the close target the close guards read. +type closeGuardRow struct { + isTemplate bool + pinned bool + status types.Status + assignee string +} + +// enforceCloseGuardsInTx applies the close guards — template read-only, the +// pin, and the assignee authority fence (be-035) — to an open target whose +// dependency routing the caller already read from this transaction. +// +// They are the guards `bd close` used to run as a pre-read in cmd/bd, which +// left every other caller of the role (bd serve, BatchCloser, BatchApplier +// close items, a library embedder) closing what bd refused. Living here, they +// read the row the close is about to write, inside the same transaction, so +// there is no read-then-write window either. +// +// Order is the historical one: template first (no bypass), then the pin, then +// the assignee — so a request that trips more than one reads the refusal bd +// always printed for it. force bypasses the pin and the assignee and nothing +// else. +// +// The caller runs this only for a target that is not already literally closed: +// a re-close has no state change to guard, and it stays the idempotent no-op it +// has always been (ga-ktn9pe.4.8) — a forced close of a pinned bead leaves the +// pin set, and the plain retry must not then refuse. +func enforceCloseGuardsInTx(ctx context.Context, tx DBTX, id, targetColumn, actor string, force bool) error { + row, err := readCloseGuardRowInTx(ctx, tx, id, targetColumn) + if err != nil { + return err + } + return CheckClosable(id, &types.Issue{ + IsTemplate: row.isTemplate, + Pinned: row.pinned, + Status: row.status, + Assignee: row.assignee, + }, actor, force) +} + +// CheckClosable is the close guards' one decision over an issue's pre-image: +// template read-only (*TemplateReadOnlyError, no bypass), then — unless force — +// the pin (*PinnedError, either spelling) and the assignee authority fence +// (*CloseNotAssigneeError; an unassigned issue is closable by anyone). A nil +// issue passes. It is pure: enforceCloseGuardsInTx applies it to the row the +// close is about to write, inside the close transaction, which is the +// authority; `bd close` also applies it to its resolved pre-image ahead of its +// own gate-satisfaction check, so a refusal that trips both reads the guard's +// sentence, as it always has. +// +// The caller decides whether to guard at all: an already-closed target is an +// idempotent re-close with no state change to guard (ga-ktn9pe.4.8). +func CheckClosable(id string, issue *types.Issue, actor string, force bool) error { + if issue == nil { + return nil + } + if issue.IsTemplate { + return &publicops.TemplateReadOnlyError{IssueID: id} + } + if force { + return nil + } + // Both pin spellings are load-bearing (ga-z3vht): Gas Town pins by + // status, Gas City by the column. See validation.NotPinned. + if issue.Pinned || issue.Status == types.StatusPinned { + return &publicops.PinnedError{IssueID: id} + } + if issue.Assignee != "" && !actorMatches(issue.Assignee, actor) { + return &publicops.CloseNotAssigneeError{IssueID: id, Assignee: issue.Assignee, Actor: actor} + } + return nil +} + +//nolint:gosec // G201: table is one of two hardcoded identifiers chosen from targetColumn. +func readCloseGuardRowInTx(ctx context.Context, tx DBTX, id, targetColumn string) (closeGuardRow, error) { + table := "issues" + switch targetColumn { + case "depends_on_issue_id": + case "depends_on_wisp_id": + table = "wisps" + default: + return closeGuardRow{}, fmt.Errorf("close guard: unsupported target column %q", targetColumn) + } + var ( + row closeGuardRow + template sql.NullBool + pinned sql.NullBool + status string + assignee sql.NullString + ) + if err := tx.QueryRowContext(ctx, + "SELECT is_template, pinned, status, assignee FROM "+table+" WHERE id = ?", id, + ).Scan(&template, &pinned, &status, &assignee); err != nil { + return closeGuardRow{}, fmt.Errorf("read close guard state for %s from %s: %w", id, table, err) + } + row.isTemplate = template.Valid && template.Bool + row.pinned = pinned.Valid && pinned.Bool + row.status = types.Status(status) + row.assignee = assignee.String + return row, nil +} diff --git a/internal/storage/issueops/close_guard_test.go b/internal/storage/issueops/close_guard_test.go new file mode 100644 index 000000000..9fa419d5b --- /dev/null +++ b/internal/storage/issueops/close_guard_test.go @@ -0,0 +1,55 @@ +package issueops + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestCheckClosable pins the close guards' one decision: template first (no +// bypass), then — unless force — the pin (both spellings) and the assignee +// fence (separator-insensitive; unassigned is anyone's). +func TestCheckClosable(t *testing.T) { + cases := []struct { + name string + issue *types.Issue + actor string + force bool + want error + }{ + {"nil", nil, "alice", false, nil}, + {"plain", &types.Issue{Status: types.StatusOpen}, "alice", false, nil}, + {"template", &types.Issue{IsTemplate: true}, "alice", false, publicops.ErrTemplateReadOnly}, + {"forced template", &types.Issue{IsTemplate: true}, "alice", true, publicops.ErrTemplateReadOnly}, + {"template outranks pin", &types.Issue{IsTemplate: true, Pinned: true}, "alice", false, publicops.ErrTemplateReadOnly}, + {"pinned column", &types.Issue{Pinned: true}, "alice", false, publicops.ErrPinned}, + {"pinned status", &types.Issue{Status: types.StatusPinned}, "alice", false, publicops.ErrPinned}, + {"forced pin", &types.Issue{Pinned: true}, "alice", true, nil}, + {"pin outranks holder", &types.Issue{Pinned: true, Assignee: "bob"}, "alice", false, publicops.ErrPinned}, + {"held", &types.Issue{Assignee: "bob"}, "alice", false, publicops.ErrNotOwner}, + {"forced held", &types.Issue{Assignee: "bob"}, "alice", true, nil}, + {"own", &types.Issue{Assignee: "alice"}, "alice", false, nil}, + {"unassigned", &types.Issue{}, "alice", false, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := CheckClosable("bd-1", tc.issue, tc.actor, tc.force) + if tc.want == nil { + if err != nil { + t.Fatalf("CheckClosable = %v, want nil", err) + } + return + } + if !errors.Is(err, tc.want) { + t.Fatalf("CheckClosable = %v, want %v", err, tc.want) + } + }) + } + var held *publicops.CloseNotAssigneeError + if err := CheckClosable("bd-1", &types.Issue{Assignee: "bob"}, "alice", false); !errors.As(err, &held) || + held.IssueID != "bd-1" || held.Assignee != "bob" || held.Actor != "alice" { + t.Fatalf("held refusal = %#v", err) + } +} diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index 44add0cea..ee77833ba 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -406,10 +406,17 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string // A status update that crosses into the done category is a close by another // name, so it answers to close policy. Running after the no-op filter keeps // a done-to-done restatement policy-free, and running after the callers' - // version and assignee preconditions keeps force away from those: it - // bypasses the two close refusals and nothing else, exactly as it does for - // `bd close`. A refusal returns here, before any write, and aborts the - // caller's transaction. + // version and assignee preconditions keeps force away from those: here it + // bypasses the two close-policy refusals and nothing else. A refusal + // returns here, before any write, and aborts the caller's transaction. + // + // Close policy is ALL it answers to. The pin guard and the assignee + // authority fence, which CloseIssueCheckedInTx applies to the close verbs, + // are deliberately not run here: a crossing still closes a pinned issue or + // a bead another actor holds on every update leg (`bd update -s closed`, + // PATCH, batchApply), as it always has. Refusing those would change what + // the update legs admit, which is its own decision and not part of putting + // the close verbs' guards in the role. crossing, err := CrossesIntoDoneCategoryInTx(ctx, tx, oldIssue.Status, updates) if err != nil { return nil, err diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 66553307f..61f80cde0 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -407,6 +407,9 @@ type Storage interface { // when the current assignee differs. UnclaimIssueIfAssignee(ctx context.Context, id string, actor string, expectedAssignee string) error UpdateIssueType(ctx context.Context, id string, issueType string, actor string) error + // CloseIssue is the raw close: it applies neither the close guards nor + // close policy, except over HTTP, where it closes with force and a template + // — the one guard force never bypasses — still refuses. CloseIssue(ctx context.Context, id string, reason string, actor string, session string) error // CloseIssueChecked closes an issue, but refuses with ErrCloseOpenChildren // when it has open parent-child dependents, or ErrCloseBlocked when it has a @@ -418,9 +421,16 @@ type Storage interface { // (no TOCTOU). When opts.ExpectedVersion is non-nil it adds an orthogonal // optimistic-concurrency precondition: the close proceeds only if the issue's // current RowVersion still equals *opts.ExpectedVersion, else it refuses with - // ErrVersionMismatch atomically (Force does NOT bypass this check). Already- - // closed is an idempotent success with Unchanged=true; a missing issue returns - // ErrNotFound. + // ErrVersionMismatch atomically (Force does NOT bypass this check). After + // the version check and before the child and blocker policy it applies the + // close guards in the same transaction: a template refuses with + // *issueops.TemplateReadOnlyError whatever Force says, and unless Force is + // set a pinned issue refuses with *issueops.PinnedError and one assigned to + // someone other than actor with *issueops.CloseNotAssigneeError. Already- + // closed is an idempotent success with Unchanged=true and skips the guards; + // a missing issue returns ErrNotFound. CloseIssue above is the raw close and + // applies neither the guards nor the policy, except that over HTTP, where + // the template guard has no bypass on the wire, a template still refuses. CloseIssueChecked(ctx context.Context, id string, actor string, opts CloseIssueOptions) (CloseIssueResult, error) DeleteIssue(ctx context.Context, id string) error SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) @@ -647,7 +657,7 @@ type Storage interface { type CloseIssueOptions struct { Reason string Session string - Force bool // bypass the is_blocked guard (mirrors `bd close --force`) + Force bool // bypass close policy and the pin and assignee guards, never the template (mirrors `bd close --force`) // ExpectedVersion, when non-nil, gates the close on an optimistic-concurrency // check: the close proceeds only if the issue's current RowVersion (the // row_lock token) equals *ExpectedVersion, otherwise it refuses with diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index 9bb82d3e9..2d8b4673d 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -163,6 +163,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("SplicesTheMetadataOfATemplateItCreates", func(t *testing.T) { conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) }) + t.Run("CloseItemsAnswerToTheCloseGuards", func(t *testing.T) { + conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) + }) } func newUOWBatchApplyFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchApplyFixture { diff --git a/internal/storage/uow/batch_closer_contract_test.go b/internal/storage/uow/batch_closer_contract_test.go index 43640e738..4ca8d2b4e 100644 --- a/internal/storage/uow/batch_closer_contract_test.go +++ b/internal/storage/uow/batch_closer_contract_test.go @@ -47,6 +47,7 @@ func TestBatchCloserContract(t *testing.T) { {name: "AllRefusedBatchRecordsNoHistory", prefix: "bc-nohistory", run: conformance.RunBatchCloserAllRefusedBatchRecordsNoHistory}, {name: "DoesNotMutateTheCallerRequest", prefix: "bc-snapshot", run: conformance.RunBatchCloserDoesNotMutateTheCallerRequest}, {name: "SettlesTheDependersOfWhatItClosed", prefix: "bc-blocked", run: conformance.RunBatchCloserSettlesTheDependersOfWhatItClosed}, + {name: "ItemsAnswerToTheCloseGuards", prefix: "bc-guard", run: conformance.RunBatchCloserItemsAnswerToTheCloseGuards}, } { t.Run(test.name, func(t *testing.T) { test.run(t, ctx, newUOWBatchCloserFixture(t, provider, test.prefix)) diff --git a/internal/storage/uow/lifecycle_close_reopen_contract_test.go b/internal/storage/uow/lifecycle_close_reopen_contract_test.go index c495de06a..9a77a7f95 100644 --- a/internal/storage/uow/lifecycle_close_reopen_contract_test.go +++ b/internal/storage/uow/lifecycle_close_reopen_contract_test.go @@ -81,6 +81,9 @@ func TestLifecycleCloseReopenContract(t *testing.T) { t.Run("ReopenReblocksItsDependers", func(t *testing.T) { conformance.RunLifecycleReopenReblocksItsDependers(t, ctx, fixture) }) + t.Run("CloseEnforcesTheCloseGuards", func(t *testing.T) { + conformance.RunLifecycleCloseEnforcesTheCloseGuards(t, ctx, fixture) + }) } func newUOWLifecycleCloseReopenFixture(t *testing.T, ctx context.Context, prefix string) conformance.LifecycleCloseReopenFixture { diff --git a/internal/validation/issue.go b/internal/validation/issue.go index bde847ec3..bad1fc61a 100644 --- a/internal/validation/issue.go +++ b/internal/validation/issue.go @@ -66,7 +66,7 @@ func NotPinned(force bool) IssueValidator { return nil // Let Exists() handle nil check if needed } if !force && (issue.Pinned || issue.Status == types.StatusPinned) { - return fmt.Errorf("cannot modify pinned issue %s (use --force to override)", id) + return &issueops.PinnedError{IssueID: id} } return nil } @@ -171,7 +171,7 @@ func AssigneeMatches(actor string, force bool) IssueValidator { if issue.Assignee == "" || ActorMatches(issue.Assignee, actor) { return nil } - return fmt.Errorf("cannot close %s: assignee is %q, actor is %q; reclaim or use --force to override", id, issue.Assignee, actor) + return &issueops.CloseNotAssigneeError{IssueID: id, Assignee: issue.Assignee, Actor: actor} } } diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index 76d301234..7b3634a08 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -167,7 +167,8 @@ type CloseItem struct { // Reason and Session are CloseRequest's, under its first-close-wins rule. Reason string Session string - // Force bypasses blocker and open-child close policy, and nothing else. + // Force is CloseRequest.Force: it bypasses blocker and open-child close + // policy, the pin and the assignee fence, and never the template guard. // // CLOSE POLICY EVALUATES AT THIS ITEM, against the row as this request has // already changed it — the same as-modified rule the update guards take. A diff --git a/issueops/batchcloser.go b/issueops/batchcloser.go index f557a791b..bd2861bb1 100644 --- a/issueops/batchcloser.go +++ b/issueops/batchcloser.go @@ -40,10 +40,11 @@ type CloseBatchRequest struct { Items []BatchCloseItem // Session records the closing session against every item. Session string - // Force bypasses only blocker and open-child close policy, for every item. - // It never bypasses validation, and it never bypasses existence: an id that - // names nothing refuses whether or not force is set. It is request-wide - // because the flag that spells it is. + // Force bypasses blocker and open-child close policy, the pin and the + // assignee fence, for every item — exactly CloseRequest.Force. It never + // bypasses the template guard or validation, and it never bypasses + // existence: an id that names nothing refuses whether or not force is set. + // It is request-wide because the flag that spells it is. // // It would not bypass a per-item LIFECYCLE PRECONDITION either, but no // batch item carries one: there is no counterpart here to the @@ -86,7 +87,8 @@ type CloseOutcome struct { OpenChildren int // Err is this item's refusal, and NEVER aborts the rest of the batch. It // carries the same typed close vocabulary Lifecycle.Close returns — - // ErrNotFound, ErrCloseBlocked, *CloseOpenChildrenError — so a caller + // ErrNotFound, ErrCloseBlocked, *CloseOpenChildrenError, + // *TemplateReadOnlyError, *PinnedError, *CloseNotAssigneeError — so a caller // classifies it with errors.Is and errors.As rather than by reading prose. Err error } diff --git a/issueops/errors.go b/issueops/errors.go index 7db6159ed..6a1dfe921 100644 --- a/issueops/errors.go +++ b/issueops/errors.go @@ -242,6 +242,51 @@ func (e *CloseOpenChildrenError) Unwrap() error { return ErrCloseOpenChildren } +// ErrPinned is returned when an unforced close names a pinned issue — pinned by +// either of the two spellings bd uses, the Pinned column or the pinned status. +// Force (CloseRequest.Force, CloseBatchRequest.Force, CloseItem.Force) bypasses +// it. +var ErrPinned = errors.New("issue is pinned") + +// PinnedError reports the pinned issue an unforced close was refused for. Its +// message is the sentence `bd close` has always printed for the refusal. +type PinnedError struct { + // IssueID names the pinned issue. + IssueID string +} + +func (e *PinnedError) Error() string { + return fmt.Sprintf("cannot modify pinned issue %s (use --force to override)", e.IssueID) +} + +// Unwrap makes PinnedError match ErrPinned. +func (e *PinnedError) Unwrap() error { return ErrPinned } + +// CloseNotAssigneeError reports an unforced close refused because the issue is +// assigned to someone other than the closing actor (be-035): storage would +// accept the id-only close, so without the guard actor A silently closes the +// bead actor B holds. Authority is identity-by-string under the same +// separator-insensitive comparison every assignee guard uses; an unassigned +// issue is closable by anyone. Force bypasses it. +// +// It matches ErrNotOwner — the ownership refusal a release earns for the same +// situation — and its message is the sentence `bd close` has always printed. +type CloseNotAssigneeError struct { + // IssueID names the issue that refused the close. + IssueID string + // Assignee is the holder observed inside the refusing transaction. + Assignee string + // Actor is the closing actor the request named. + Actor string +} + +func (e *CloseNotAssigneeError) Error() string { + return fmt.Sprintf("cannot close %s: assignee is %q, actor is %q; reclaim or use --force to override", e.IssueID, e.Assignee, e.Actor) +} + +// Unwrap makes CloseNotAssigneeError match ErrNotOwner. +func (e *CloseNotAssigneeError) Unwrap() error { return ErrNotOwner } + // ErrAlreadyExists is returned when a create operation is given an ID that is // already occupied. The issue and wisp tables share one ID space. var ErrAlreadyExists = errors.New("issue already exists") diff --git a/issueops/issueops.go b/issueops/issueops.go index 69c8a20a1..3d5588e2b 100644 --- a/issueops/issueops.go +++ b/issueops/issueops.go @@ -362,8 +362,10 @@ type CloseRequest struct { // and read back on CloseResult.Issue.ClosedBySession, under the same // first-close-wins rule as Reason. Session string - // Force bypasses only blocker and open-child close policy. It never bypasses - // validation, ExpectedVersion, or lifecycle rules. + // Force bypasses blocker and open-child close policy and the two close + // guards a force has always waived in bd: the pin, and the assignee + // authority fence. It never bypasses the template guard, validation, + // ExpectedVersion, or lifecycle rules. Force bool // ExpectedVersion requires the current row version to match, and is checked // BEFORE the idempotent close — so a re-close of an already-closed issue @@ -496,10 +498,16 @@ type Lifecycle interface { // Close validates guards and commits the complete request as one atomic // mutation. It moves the issue to literal StatusClosed, including from a // configured done status. ExpectedVersion is checked first, including for an - // idempotent close. An unforced close with open children returns - // CloseOpenChildrenError without mutation. Force bypasses blocker and - // open-child policy and reports OpenChildren, including for an idempotent - // re-close. A refusal or validation error leaves persistent state unchanged. + // idempotent close. A close of an issue that is not already closed answers + // to the close guards before close policy: a template returns + // *TemplateReadOnlyError, forced or not; unforced, a pinned issue returns + // *PinnedError and one assigned to someone other than Actor (compared + // separator-insensitively) returns *CloseNotAssigneeError, matching + // ErrNotOwner. An unforced close with open children returns + // CloseOpenChildrenError without mutation. Force bypasses the pin, the + // assignee fence, and blocker and open-child policy, and reports + // OpenChildren, including for an idempotent re-close. A refusal or + // validation error leaves persistent state unchanged. Close(context.Context, CloseRequest) (CloseResult, error) // Reopen validates guards and commits the complete request as one atomic // mutation. It moves literal StatusClosed and configured done statuses to diff --git a/issueops/template_errors.go b/issueops/template_errors.go index 8997324c3..73dc50501 100644 --- a/issueops/template_errors.go +++ b/issueops/template_errors.go @@ -6,8 +6,9 @@ import ( ) // ErrTemplateReadOnly is returned when a mutation that guards templates names -// one. Every update does (Lifecycle.Update, a BatchApplier update item); not -// every verb does yet (bd-jkp9v3). Templates are read-only: the way to get +// one. Every update does (Lifecycle.Update, a BatchApplier update item), and so +// does every close (Lifecycle.Close, BatchCloser, a BatchApplier close item); +// not every verb does yet (bd-jkp9v3). Templates are read-only: the way to get // work out of one is to pour it (`bd mol pour`), which creates new issues and // leaves the template untouched. There is no force bypass — no Force flag // waives it; only an update that sets UpdateRequest.AllowTemplate (bd label, diff --git a/tests/oracle-a/harness/scenarios/enumerated.json b/tests/oracle-a/harness/scenarios/enumerated.json index 242e11e03..3135e819e 100644 --- a/tests/oracle-a/harness/scenarios/enumerated.json +++ b/tests/oracle-a/harness/scenarios/enumerated.json @@ -67,7 +67,7 @@ "--json" ] ], - "pins": "close.go has no already-closed guard in validateIssueClosable (only NotTemplate+NotPinned); second close re-runs CloseIssue; pins idempotency/event behavior of the storage layer.", + "pins": "the close guards (storeissueops.CheckClosable: template, pin, assignee) skip a row already at literal closed (ga-ktn9pe.4.8), so the second close reaches the role as its idempotent re-close (Changed=false, exit 0); pins idempotency/event behavior of the storage layer.", "deterministic": true }, { @@ -598,7 +598,7 @@ "--json" ] ], - "pins": "validateIssueClosable NotPinned(force): without force stderr 'cannot modify pinned issue clp-1 (use --force to override)' + SilentExit; with --force closes.", + "pins": "the pin close guard (storeissueops.CheckClosable, waived by force; the role applies it again inside the close transaction): without force stderr 'cannot modify pinned issue clp-1 (use --force to override)' + SilentExit; with --force closes.", "deterministic": true }, { From 1531a31760c44fc4b3a264d6b263e608247f9b79 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 19:20:33 +0000 Subject: [PATCH 020/151] ci(bazel): halve the embedded cmd tier's cost (seeded bd init schema, no race exit sleep, concurrency-aware packing) (#7437) //cmd/bd:bd_embedded_test sits at Bazel's 50-shard cap and was the embedded lane's floor (shards 62-250s). Measured on rbe-west, every shard's wall time was its single slowest top-level test, and most of a test's time was `bd init` (~20s under -race: one Dolt commit per schema migration) plus a fixed 1s per bd subprocess exit (the race runtime's atexit_sleep_ms). - bdInit seeds .beads/embeddeddolt/ from a per-process schema template (an empty database with every migration applied, made once per shard by a non-race build of internal/storage/embeddeddolt/cmd) before running the real `bd init`, which then skips the migration chain and does everything else itself (identity, prefix, config, hooks, agent files, git commit). Only for argument lists it understands (--prefix/--database plus --quiet/--skip-hooks/--skip-agents/--non-interactive), and it checks the result is the seeded embedded database. BEADS_TEST_FRESH_INIT=1 turns it off. The TestEmbeddedInit* tests run their own init and are unaffected. - GORACE=atexit_sleep_ms=0 on the target: the bd subprocesses no longer sleep 1s before exiting. A race they hit while running still fails the command as before. - TestEmbeddedInitSelectedExcludeRouting (serial, t.Setenv, 7 full inits, ~113s) is split into ...Stealth and ...Fork top-level functions; same cases, same assertions. - The generator packs with a model of the shard process (pack_concurrent): tests without a top-level t.Parallel run one after another, the rest share -test.parallel=4. The durations are re-measured on rbe-west. rbe-west, `bazel test //cmd/bd:bd_embedded_test --config=embedded --nocache_test_results`: test-seconds 20.2k -> ~10.3k, shard-seconds 8.0k -> ~4.2k, slowest shard 243s -> 138s / 163s (two runs, before the repack). Bead: ga-vnycm2.17 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/embedded-cmd-test-shards.txt | 499 +++++++-------- .github/workflows/bazel.yml | 16 +- cmd/bd/BUILD.bazel | 11 + cmd/bd/embedded_init_template_test.go | 239 ++++++++ cmd/bd/init_embedded_test.go | 26 +- internal/storage/embeddeddolt/cmd/BUILD.bazel | 18 + scripts/ci/_embedded_shard_manifest_lib.py | 33 +- scripts/ci/embedded_cmd_test_durations.json | 579 +++++++++--------- scripts/ci/gen_embedded_cmd_shard_manifest.py | 38 +- scripts/pr_risk_bazel_coverage_test.go | 7 +- scripts/shard_manifest_test.go | 29 + 11 files changed, 929 insertions(+), 566 deletions(-) create mode 100644 cmd/bd/embedded_init_template_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index a0ef80ddb..486e904fe 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -188,9 +188,11 @@ # # 50-shard split for the Bazel-only embedded-Dolt cmd tier # (bazel-embedded in .github/workflows/bazel.yml), bin-packed -# longest-processing-time-first by measured wall-time from +# longest-first by measured wall-time from # scripts/ci/embedded_cmd_test_durations.json (see that file for provenance -# and its "relative weight, not absolute SLA" caveat). +# and its "relative weight, not absolute SLA" caveat) onto the shard +# whose estimated wall time grows least: serial tests (no top-level +# t.Parallel) add up, parallel ones share -test.parallel=4. # Regenerate with scripts/ci/gen_embedded_cmd_shard_manifest.py 50 # --weights=duration --write after adding, splitting or removing # TestEmbedded* functions in cmd/bd/*_embedded_test.go. --write is @@ -206,301 +208,302 @@ # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator or the durations file instead. -50 1 TestEmbeddedCommentsCLI -50 1 TestEmbeddedDepAddWispSource -50 1 TestEmbeddedListFiltering -50 1 TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget +50 1 TestEmbeddedInitA -50 2 TestEmbeddedCreateRepoFromNonBeadsCwd -50 2 TestEmbeddedDeleteJSONDependencyErrorContract -50 2 TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate -50 2 TestEmbeddedHistoryConcurrent -50 2 TestEmbeddedInitA +50 2 TestEmbeddedGateCreate -50 3 TestEmbeddedAdminCompactTier2Honesty -50 3 TestEmbeddedMaxRowsListLimitInteraction -50 3 TestEmbeddedOrphans -50 3 TestEmbeddedShowUnresolvableDeps -50 3 TestEmbeddedStale +50 3 TestEmbeddedRoutedMutatingSiblingAssignTagDep -50 4 TestEmbeddedBootstrapConcurrent -50 4 TestEmbeddedDuplicatesConcurrent -50 4 TestEmbeddedInfo -50 4 TestEmbeddedMaxRowsDoctorAndConfig -50 4 TestEmbeddedRoutedSiblingWritesCommitTargetHead +50 4 TestEmbeddedUpdateConcurrent -50 5 TestEmbeddedCreateScalarAndRelationships -50 5 TestEmbeddedDefer -50 5 TestEmbeddedDoltBlockedAndCommit -50 5 TestEmbeddedGCConcurrent -50 5 TestEmbeddedPromote +50 5 TestEmbeddedMaxRowsListLimitInteraction -50 6 TestEmbeddedCreateStorageClassConflicts -50 6 TestEmbeddedGC -50 6 TestEmbeddedGetDependencies -50 6 TestEmbeddedQuickConcurrent +50 6 TestEmbeddedSwarm -50 7 TestEmbeddedCreateCrossRepoDryRunWithParent -50 7 TestEmbeddedEpicStatus -50 7 TestEmbeddedGate -50 7 TestEmbeddedWhere +50 7 TestEmbeddedDepA -50 8 TestEmbeddedChangeDirOverridesInheritedBeadsDir -50 8 TestEmbeddedMemoryConcurrent -50 8 TestEmbeddedUpdateConcurrent -50 8 TestEmbeddedVCConcurrent +50 8 TestEmbeddedDepB -50 9 TestEmbeddedCloseAlreadyClosed -50 9 TestEmbeddedConfigConcurrent -50 9 TestEmbeddedExternalCapabilityReadyAfterShip -50 9 TestEmbeddedPrimeConcurrent +50 9 TestEmbeddedKVConcurrent -50 10 TestEmbeddedDepB -50 10 TestEmbeddedHuman -50 10 TestEmbeddedInitExplicitBeadsDirIsAuthoritative -50 10 TestEmbeddedPromoteCLI +50 10 TestEmbeddedListFiltering -50 11 TestEmbeddedBlockedConcurrent -50 11 TestEmbeddedDepA -50 11 TestEmbeddedMaxRowsDepTreeAndDuplicates -50 11 TestEmbeddedState +50 11 TestEmbeddedInitRoleRouting -50 12 TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead -50 12 TestEmbeddedPrimeChangeDirPrimesTarget -50 12 TestEmbeddedPromoteCLIConcurrent -50 12 TestEmbeddedSearchConcurrent -50 12 TestEmbeddedTypes +50 12 TestEmbeddedContributorCreate -50 13 TestEmbeddedAutoImportJSONLNoExplicitCommit -50 13 TestEmbeddedDiff -50 13 TestEmbeddedDuplicates -50 13 TestEmbeddedShipConcurrent +50 13 TestEmbeddedInitGitBootstrapRouting -50 14 TestEmbeddedInitConcurrent -50 14 TestEmbeddedOrphansConcurrent -50 14 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget -50 14 TestEmbeddedRoutedMutatingSiblingDeleteClose -50 14 TestEmbeddedSwarm +50 14 TestEmbeddedCloseConcurrent -50 15 TestEmbeddedChildrenConcurrent -50 15 TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout -50 15 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget -50 15 TestEmbeddedEpicCloseEligibleReasonFlag -50 15 TestEmbeddedExplicitHierarchicalParentEdges -50 15 TestEmbeddedIfRevisionReopenGuardDirect -50 15 TestEmbeddedVersionConcurrent +50 15 TestEmbeddedCloseAlreadyClosed -50 16 TestEmbeddedBlockedLabelFilters -50 16 TestEmbeddedGCConditionalMatcherDecode -50 16 TestEmbeddedGateInheritsDownParentChain -50 16 TestEmbeddedGatedRendering -50 16 TestEmbeddedKV -50 16 TestEmbeddedMaxRowsReady -50 16 TestEmbeddedMemoryCorpusBudget -50 16 TestEmbeddedOpen_EmptyDatabaseRejected -50 16 TestEmbeddedRestoreConcurrent -50 16 TestEmbeddedVCCommitVariantsAndMerge -50 16 TestEmbeddedVersion +50 16 TestEmbeddedRoutedSiblingWritesCommitTargetHead -50 17 TestEmbeddedCloseGuardsAndEpics -50 17 TestEmbeddedCompact -50 17 TestEmbeddedGateConcurrent -50 17 TestEmbeddedListComments -50 17 TestEmbeddedReadOnlyCommandsSpawnNoGit +50 17 TestEmbeddedConfigConcurrent -50 18 TestEmbeddedContext -50 18 TestEmbeddedContributorCreate -50 18 TestEmbeddedImportClosedChildWhoseParentWasDeleted -50 18 TestEmbeddedReopen -50 18 TestEmbeddedStateConcurrent +50 18 TestEmbeddedGate -50 19 TestEmbeddedAdminBlocked -50 19 TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected -50 19 TestEmbeddedDoltConcurrent -50 19 TestEmbeddedMaxRowsListBasic -50 19 TestEmbeddedPruneSafetyAndScope +50 19 TestEmbeddedShowDetailAndCurrent -50 20 TestEmbeddedCreateStorageClassBatchAndMisc -50 20 TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge -50 20 TestEmbeddedKVConcurrent -50 20 TestEmbeddedSetup +50 20 TestEmbeddedDelete -50 21 TestEmbeddedBackup -50 21 TestEmbeddedCreateCrossRepoUninit -50 21 TestEmbeddedDiffConcurrent -50 21 TestEmbeddedQuick -50 21 TestEmbeddedQuickstart +50 21 TestEmbeddedBootstrap -50 22 TestEmbeddedAdminCompactReadOnlyAllowed -50 22 TestEmbeddedCreateRepoProxiedTargetError -50 22 TestEmbeddedDuplicateConcurrent -50 22 TestEmbeddedFlatten -50 22 TestEmbeddedTypesAgreeWithCreate +50 22 TestEmbeddedUnclaim -50 23 TestEmbeddedExportConcurrent -50 23 TestEmbeddedHistory -50 23 TestEmbeddedMaxRowsGraph -50 23 TestEmbeddedSetupConcurrent -50 23 TestEmbeddedShip +50 23 TestEmbeddedBootstrapConcurrent +50 23 TestEmbeddedHooksConcurrent +50 23 TestEmbeddedShowConcurrent -50 24 TestEmbeddedCloseBasic -50 24 TestEmbeddedImportUpsertAndPrefixSync -50 24 TestEmbeddedQuery -50 24 TestEmbeddedReclaimRunsTheLeaseReclaimerRole -50 24 TestEmbeddedUpdateRoutedStoreCommitsTargetHead +50 24 TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll +50 24 TestEmbeddedBackupConcurrent +50 24 TestEmbeddedBlocked +50 24 TestEmbeddedComments +50 24 TestEmbeddedCompactMisdatedChild +50 24 TestEmbeddedContextConcurrent +50 24 TestEmbeddedCreateCommitPending +50 24 TestEmbeddedCreateRepoFromNonBeadsCwd +50 24 TestEmbeddedCreateStorageClassBasic +50 24 TestEmbeddedCreateWithGitRemote +50 24 TestEmbeddedFindDuplicatesConcurrent +50 24 TestEmbeddedHistoryConcurrent +50 24 TestEmbeddedHooks +50 24 TestEmbeddedHumanConcurrent +50 24 TestEmbeddedInfoSuppressGitHooks +50 24 TestEmbeddedMigrateConcurrent +50 24 TestEmbeddedOnboard +50 24 TestEmbeddedPromote +50 24 TestEmbeddedQuickstart +50 24 TestEmbeddedWhere -50 25 TestEmbeddedBackupConcurrent -50 25 TestEmbeddedCreateStorageClassBasic -50 25 TestEmbeddedFindDuplicatesConcurrent -50 25 TestEmbeddedInitGitBootstrapRouting -50 25 TestEmbeddedInitHooksRouting +50 25 TestEmbeddedAdminCompactReadOnlyAllowed +50 25 TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber +50 25 TestEmbeddedConfigNoWorkspaceStoreFreeCommands +50 25 TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout +50 25 TestEmbeddedCreateGraphAndBulk +50 25 TestEmbeddedDoltRemoteAndPush +50 25 TestEmbeddedGateInheritsDownParentChain +50 25 TestEmbeddedListConcurrent +50 25 TestEmbeddedOnboardConcurrent +50 25 TestEmbeddedPrime +50 25 TestEmbeddedQuickstartConcurrent +50 25 TestEmbeddedRenamePrefixConcurrent +50 25 TestEmbeddedSetup +50 25 TestEmbeddedSetupConcurrent +50 25 TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead +50 25 TestEmbeddedUpdateFields +50 25 TestEmbeddedVCCommitVariantsAndMerge +50 25 TestEmbeddedVersion +50 25 TestEmbeddedVersionConcurrent -50 26 TestEmbeddedCreateCommitPending -50 26 TestEmbeddedInitC -50 26 TestEmbeddedRenamePrefixBasic -50 26 TestEmbeddedRoutedMutatingSiblingAssignTagDep -50 26 TestEmbeddedSupersedeConcurrent +50 26 TestEmbeddedAdminBlocked +50 26 TestEmbeddedBlockedLabelFilters +50 26 TestEmbeddedClaimPools +50 26 TestEmbeddedCompactConcurrent +50 26 TestEmbeddedExportConcurrent +50 26 TestEmbeddedGCConcurrent +50 26 TestEmbeddedGraphConcurrent +50 26 TestEmbeddedPrimeChangeDirPrimesTarget +50 26 TestEmbeddedPrimeConcurrent +50 26 TestEmbeddedReopenConcurrent +50 26 TestEmbeddedTodoConcurrent +50 26 TestEmbeddedTypes +50 26 TestEmbeddedTypesConcurrent -50 27 TestEmbeddedComments -50 27 TestEmbeddedMaxRowsReadyMerge -50 27 TestEmbeddedPrime -50 27 TestEmbeddedPruneDryRunAndFilters -50 27 TestEmbeddedReopenConcurrent +50 27 TestEmbeddedBranch +50 27 TestEmbeddedContext +50 27 TestEmbeddedCountIncludeEphemeral +50 27 TestEmbeddedCreateFormCommitsLabelOnlyCreate +50 27 TestEmbeddedFederationConcurrent +50 27 TestEmbeddedGateListScoped +50 27 TestEmbeddedIfRevisionReopenGuardDirect +50 27 TestEmbeddedOrphans +50 27 TestEmbeddedPurgeWispsPlaneRetention +50 27 TestEmbeddedReclaimRunsTheLeaseReclaimerRole +50 27 TestEmbeddedStaleLabelFilters -50 28 TestEmbeddedConfigNoWorkspaceStoreFreeCommands -50 28 TestEmbeddedEpicCloseEligibleReasonOutput -50 28 TestEmbeddedMigrate +50 28 TestEmbeddedAutoImportJSONLConcurrentWriter +50 28 TestEmbeddedAutoImportJSONLSkipsNonEmpty +50 28 TestEmbeddedBranchConcurrent +50 28 TestEmbeddedCreateConcurrent +50 28 TestEmbeddedEpicCloseEligibleReasonFlag +50 28 TestEmbeddedFederationStatus +50 28 TestEmbeddedLabelAddRemove +50 28 TestEmbeddedPurgeConcurrent +50 28 TestEmbeddedReadyClaimConcurrent 50 28 TestEmbeddedStatus -50 28 TestEmbeddedSwarmConcurrent -50 29 TestEmbeddedClaimPools -50 29 TestEmbeddedCreateGraphAndBulk -50 29 TestEmbeddedCreateJSONEchoesSubSecondTimestamps -50 29 TestEmbeddedHumanConcurrent -50 29 TestEmbeddedInitSelectedExcludeRouting +50 29 TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate +50 29 TestEmbeddedDepConcurrent +50 29 TestEmbeddedFlatten +50 29 TestEmbeddedGetDependencies +50 29 TestEmbeddedImportUpsertAndPrefixSync +50 29 TestEmbeddedInfo +50 29 TestEmbeddedMaxRowsDoctorAndConfig +50 29 TestEmbeddedOrphansConcurrent +50 29 TestEmbeddedRenamePrefixEdgeCases -50 30 TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll -50 30 TestEmbeddedDepConcurrent -50 30 TestEmbeddedExportFormats -50 30 TestEmbeddedReadyClaimConcurrent -50 30 TestEmbeddedStaleConcurrent +50 30 TestEmbeddedAutoImportJSONLNoExplicitCommit +50 30 TestEmbeddedCreateJSONEchoesSubSecondTimestamps +50 30 TestEmbeddedDepAddWispSource +50 30 TestEmbeddedExportFlags +50 30 TestEmbeddedListComments +50 30 TestEmbeddedPurgeBasics +50 30 TestEmbeddedReadOnlyCommandsSpawnNoGit +50 30 TestEmbeddedReadyConcurrent +50 30 TestEmbeddedRoutedMutatingSiblingDeleteClose -50 31 TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber -50 31 TestEmbeddedAutoImportJSONLConcurrentWriter -50 31 TestEmbeddedDeferAutoWake -50 31 TestEmbeddedExternalMutationPolicy -50 31 TestEmbeddedShowConcurrent +50 31 TestEmbeddedCreateCrossRepoUninit +50 31 TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget +50 31 TestEmbeddedDoltBlockedAndCommit +50 31 TestEmbeddedDoltConcurrent +50 31 TestEmbeddedGatedRendering +50 31 TestEmbeddedImportFromSources +50 31 TestEmbeddedInfoConcurrent +50 31 TestEmbeddedInitB +50 31 TestEmbeddedMigrate -50 32 TestEmbeddedCreateCrossRepo -50 32 TestEmbeddedDuplicate -50 32 TestEmbeddedInitArtifactRouting -50 32 TestEmbeddedShowBasicsAndJSON -50 32 TestEmbeddedUpdateLifecycle +50 32 TestEmbeddedBackup +50 32 TestEmbeddedExternalCapabilityReadyAfterShip +50 32 TestEmbeddedFederationPeers +50 32 TestEmbeddedLabelConcurrent +50 32 TestEmbeddedMaxRowsReadyMerge +50 32 TestEmbeddedPreviewDoesNotConsumeVersionMarker +50 32 TestEmbeddedRestore +50 32 TestEmbeddedTypesAgreeWithCreate -50 33 TestEmbeddedCreateConcurrent -50 33 TestEmbeddedCreateRepoRelativeUninitRefused -50 33 TestEmbeddedEpicCloseEligible -50 33 TestEmbeddedSearch -50 33 TestEmbeddedTodo +50 33 TestEmbeddedBlockedConcurrent +50 33 TestEmbeddedDefer +50 33 TestEmbeddedGC +50 33 TestEmbeddedPruneDryRunAndFilters +50 33 TestEmbeddedPurgeFiltered +50 33 TestEmbeddedQuery +50 33 TestEmbeddedVCConcurrent +50 33 TestEmbeddedWhereConcurrent -50 34 TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget -50 34 TestEmbeddedGateListScoped -50 34 TestEmbeddedLintConcurrent -50 34 TestEmbeddedPurgeWispsPlaneRetention -50 34 TestEmbeddedReady +50 34 TestEmbeddedCompact +50 34 TestEmbeddedCreateCrossRepoWithParent +50 34 TestEmbeddedCreateRepoRelativeUninitRefused +50 34 TestEmbeddedCreateStorageClassConflicts +50 34 TestEmbeddedDuplicatesConcurrent +50 34 TestEmbeddedEpicCloseEligibleReasonOutput +50 34 TestEmbeddedFlattenConcurrent +50 34 TestEmbeddedOpen_EmptyDatabaseRejected +50 34 TestEmbeddedRestoreConcurrent +50 34 TestEmbeddedUpdateLifecycle -50 35 TestEmbeddedCommentsCLIConcurrent -50 35 TestEmbeddedHooksConcurrent -50 35 TestEmbeddedLabelPropagateRename -50 35 TestEmbeddedListSearchAndOutput -50 35 TestEmbeddedSupersede +50 35 TestEmbeddedCreateCrossRepo +50 35 TestEmbeddedDeleteJSONDependencyErrorContract +50 35 TestEmbeddedDuplicateConcurrent +50 35 TestEmbeddedInitArtifactRouting +50 35 TestEmbeddedMaxRowsListBasic +50 35 TestEmbeddedStateConcurrent +50 35 TestEmbeddedVCStatusAndCommit -50 36 TestEmbeddedDeferConcurrent -50 36 TestEmbeddedFederationPeers -50 36 TestEmbeddedFlattenConcurrent -50 36 TestEmbeddedStatusConcurrent -50 36 TestEmbeddedTodoConcurrent +50 36 TestEmbeddedAdminCompactTier2Honesty +50 36 TestEmbeddedCloseBasic +50 36 TestEmbeddedConfig +50 36 TestEmbeddedCountConcurrent +50 36 TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead +50 36 TestEmbeddedInitConcurrent +50 36 TestEmbeddedLintConcurrent +50 36 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget -50 37 TestEmbeddedInfoSuppressGitHooks -50 37 TestEmbeddedInitB +50 37 TestEmbeddedChangeDirOverridesInheritedBeadsDir +50 37 TestEmbeddedChildren +50 37 TestEmbeddedChildrenConcurrent +50 37 TestEmbeddedCreateCrossRepoDryRunWithParent +50 37 TestEmbeddedCreateStorageClassBatchAndMisc 50 37 TestEmbeddedQueryConcurrent -50 37 TestEmbeddedUndeferConcurrent -50 37 TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead +50 37 TestEmbeddedStaleConcurrent -50 38 TestEmbeddedBlocked -50 38 TestEmbeddedCreateCrossRepoWithParent -50 38 TestEmbeddedEpicConcurrent -50 38 TestEmbeddedHooks -50 38 TestEmbeddedUndefer +50 38 TestEmbeddedCreateDryRunDoesNotMigrate +50 38 TestEmbeddedImportPromotedWispRoundtrip +50 38 TestEmbeddedMaxRowsReady +50 38 TestEmbeddedReady +50 38 TestEmbeddedRenamePrefixBasic +50 38 TestEmbeddedShip +50 38 TestEmbeddedSupersede -50 39 TestEmbeddedBranchConcurrent -50 39 TestEmbeddedCountIncludeInfra -50 39 TestEmbeddedCreateWithGitRemote -50 39 TestEmbeddedDelete -50 39 TestEmbeddedUpdateFields +50 39 TestEmbeddedDeferAutoWake +50 39 TestEmbeddedEpicConcurrent +50 39 TestEmbeddedGateConcurrent +50 39 TestEmbeddedInitC +50 39 TestEmbeddedKV +50 39 TestEmbeddedLint +50 39 TestEmbeddedPromoteCLI -50 40 TestEmbeddedCountConcurrent -50 40 TestEmbeddedCountIncludeEphemeral -50 40 TestEmbeddedCreateFormCommitsLabelOnlyCreate -50 40 TestEmbeddedLabelEditReports -50 40 TestEmbeddedPreviewDoesNotConsumeVersionMarker +50 40 TestEmbeddedCommentsCLI +50 40 TestEmbeddedCommentsCLIConcurrent +50 40 TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead +50 40 TestEmbeddedDiffConcurrent +50 40 TestEmbeddedImportConcurrent +50 40 TestEmbeddedSupersedeConcurrent +50 40 TestEmbeddedTodo -50 41 TestEmbeddedGateCreate -50 41 TestEmbeddedLabelAddRemove -50 41 TestEmbeddedLabelConcurrent -50 41 TestEmbeddedListConcurrent -50 41 TestEmbeddedTypesConcurrent +50 41 TestEmbeddedHistory +50 41 TestEmbeddedInitSelectedExcludeRoutingFork +50 41 TestEmbeddedListSearchAndOutput +50 41 TestEmbeddedPromoteCLIConcurrent +50 41 TestEmbeddedQuick +50 41 TestEmbeddedUndefer -50 42 TestEmbeddedImportFromSources -50 42 TestEmbeddedReadyConcurrent -50 42 TestEmbeddedRenamePrefixEdgeCases -50 42 TestEmbeddedRestore -50 42 TestEmbeddedSquashWispMoleculeClearsRootEphemeral +50 42 TestEmbeddedCountIncludeInfra +50 42 TestEmbeddedDuplicate +50 42 TestEmbeddedEpicCloseEligible +50 42 TestEmbeddedInitExplicitBeadsDirIsAuthoritative +50 42 TestEmbeddedQuietFlagSuppressesSuccessOutput +50 42 TestEmbeddedStatusConcurrent -50 43 TestEmbeddedFindDuplicates -50 43 TestEmbeddedGraph -50 43 TestEmbeddedImportPromotedWispRoundtrip -50 43 TestEmbeddedInfoConcurrent -50 43 TestEmbeddedQuickstartConcurrent +50 43 TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected +50 43 TestEmbeddedImportClosedChildWhoseParentWasDeleted +50 43 TestEmbeddedLabelEditReports +50 43 TestEmbeddedShowBasicsAndJSON +50 43 TestEmbeddedState +50 43 TestEmbeddedSwarmConcurrent -50 44 TestEmbeddedDoltRemoteAndPush -50 44 TestEmbeddedGraphConcurrent -50 44 TestEmbeddedImportConcurrent -50 44 TestEmbeddedInitRoleRouting -50 44 TestEmbeddedWhereConcurrent +50 44 TestEmbeddedFindDuplicates +50 44 TestEmbeddedMaxRowsGraph +50 44 TestEmbeddedMemory +50 44 TestEmbeddedShipConcurrent +50 44 TestEmbeddedShowUnresolvableDeps +50 44 TestEmbeddedSquashWispMoleculeClearsRootEphemeral -50 45 TestEmbeddedChildren -50 45 TestEmbeddedCompactConcurrent -50 45 TestEmbeddedMigrateConcurrent -50 45 TestEmbeddedShowDetailAndCurrent -50 45 TestEmbeddedVCStatusAndCommit +50 45 TestEmbeddedCloseGuardsAndEpics +50 45 TestEmbeddedCreateRepoProxiedTargetError +50 45 TestEmbeddedCreateScalarAndRelationships +50 45 TestEmbeddedDuplicates +50 45 TestEmbeddedUndeferConcurrent +50 45 TestEmbeddedUpdateRoutedStoreCommitsTargetHead -50 46 TestEmbeddedCompactMisdatedChild 50 46 TestEmbeddedCount -50 46 TestEmbeddedCreateDryRunDoesNotMigrate -50 46 TestEmbeddedFederationStatus -50 46 TestEmbeddedStaleLabelFilters +50 46 TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge +50 46 TestEmbeddedDepNoCycleCheck +50 46 TestEmbeddedEpicStatus +50 46 TestEmbeddedInitHooksRouting +50 46 TestEmbeddedReopen -50 47 TestEmbeddedBootstrap -50 47 TestEmbeddedBranch -50 47 TestEmbeddedContextConcurrent -50 47 TestEmbeddedLint -50 47 TestEmbeddedPurgeConcurrent +50 47 TestEmbeddedDiff +50 47 TestEmbeddedGraph +50 47 TestEmbeddedHuman +50 47 TestEmbeddedMemoryCorpusBudget +50 47 TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget +50 47 TestEmbeddedQuickConcurrent +50 47 TestEmbeddedSearchConcurrent -50 48 TestEmbeddedCloseConcurrent -50 48 TestEmbeddedConfig -50 48 TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead -50 48 TestEmbeddedOnboardConcurrent -50 48 TestEmbeddedQuietFlagSuppressesSuccessOutput +50 48 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget +50 48 TestEmbeddedExportFormats +50 48 TestEmbeddedLabelPropagateRename +50 48 TestEmbeddedMaxRowsDepTreeAndDuplicates +50 48 TestEmbeddedMemoryConcurrent +50 48 TestEmbeddedSearch -50 49 TestEmbeddedDepNoCycleCheck -50 49 TestEmbeddedFederationConcurrent -50 49 TestEmbeddedMemory -50 49 TestEmbeddedOnboard -50 49 TestEmbeddedPurgeFiltered +50 49 TestEmbeddedExplicitHierarchicalParentEdges +50 49 TestEmbeddedInitSelectedExcludeRoutingStealth -50 50 TestEmbeddedAutoImportJSONLSkipsNonEmpty -50 50 TestEmbeddedExportFlags -50 50 TestEmbeddedPurgeBasics -50 50 TestEmbeddedRenamePrefixConcurrent -50 50 TestEmbeddedUnclaim +50 50 TestEmbeddedDeferConcurrent +50 50 TestEmbeddedExternalMutationPolicy +50 50 TestEmbeddedGCConditionalMatcherDecode +50 50 TestEmbeddedPruneSafetyAndScope +50 50 TestEmbeddedStale diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 1be23c9fa..e434e70d3 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1301,12 +1301,16 @@ jobs: # cap per sh_test-like rule ("Having more than 50 shards is indicative of # poor test organization. Please reduce the number of shards." -- a # native analysis-time error, not a style lint); on rbe-west its shards - # ran 83-209s (6.5k shard-seconds, 2026-10-09), so it, not storage or - # //internal/httpclient:httpclient_served_test, is this lane's floor. - # Lowering it needs per-test speedups spread across the suite or - # splitting bd_embedded_test into multiple <=50-shard sh_test rules with - # shard-index-offset plumbing through tools/bazel/go_test_manifest_shard.sh - # and check_shard_coverage.py. On RBE the tier's wall time ~= that floor + # ran 83-209s (6.5k shard-seconds, 2026-10-09) until bdInit seeded each + # workspace from a per-process schema template and the race bd + # subprocesses stopped sleeping 1s at exit (~4.1-4.4k shard-seconds, + # slowest shard ~140-160s, its slowest single tests ~100-130s under + # load), so it, not storage or //internal/httpclient:httpclient_served_test, + # is still this lane's floor. Lowering it further needs per-test speedups + # in its slowest functions; more shards (splitting bd_embedded_test into + # multiple <=50-shard sh_test rules with shard-index-offset plumbing + # through tools/bazel/go_test_manifest_shard.sh and + # check_shard_coverage.py) would not move the slowest single test. On RBE the tier's wall time ~= that floor # + fixed cost (checkout, build, shard dispatch), not ~= # sum(durations)/shard_count the way a CPU-core-bound local sweep is -- # each shard is its own remote action and runs concurrently with every diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index f725268ee..00cf8b088 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -731,6 +731,7 @@ go_test( "duplicates_embedded_test.go", "duplicates_proxied_integration_test.go", "duplicates_test.go", + "embedded_init_template_test.go", "envfile_test.go", "epic_embedded_test.go", "epic_proxied_integration_test.go", @@ -1432,12 +1433,22 @@ sh_test( ":go_srcs", "//:.github/scripts/embedded-cmd-test-shards.txt", "//:.github/scripts/embedded-test-shard.sh", + "//internal/storage/embeddeddolt/cmd:cmd_norace", "@go_sdk//:bin/gofmt", ], env = { "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", + # bdInit's schema template (embedded_init_template_test.go), migrated + # by a non-race build of the embeddeddolt open tool. + "BEADS_TEST_EMBEDDED_SCHEMA_TOOL": "$(rlocationpath //internal/storage/embeddeddolt/cmd:cmd_norace)", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + # The race runtime sleeps 1s before every process exit by default + # (atexit_sleep_ms) so goroutines still running can report a race; + # the bd subprocesses (hundreds per shard, each a few seconds) exit + # through it every time. Races they hit while running still fail + # the command (exit 66) as before. + "GORACE": "atexit_sleep_ms=0", }, shard_count = 50, tags = ["embedded"], diff --git a/cmd/bd/embedded_init_template_test.go b/cmd/bd/embedded_init_template_test.go new file mode 100644 index 000000000..8f57dff16 --- /dev/null +++ b/cmd/bd/embedded_init_template_test.go @@ -0,0 +1,239 @@ +//go:build cgo + +package main + +import ( + "context" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +// freshInitEnv makes bdInit leave every workspace's schema to its own +// `bd init` instead of seeding it from the per-process schema template +// (seedEmbeddedSchema), to check that a failure does not depend on the seed. +const freshInitEnv = "BEADS_TEST_FRESH_INIT" + +// schemaTemplateDatabase is the template's database directory name; each +// seed is renamed to the workspace's own database name. +const schemaTemplateDatabase = "schematemplate" + +var ( + schemaTemplateOnce sync.Once + schemaTemplateDir string + schemaTemplateErr error +) + +// schemaToolEnv names a non-race build of +// internal/storage/embeddeddolt/cmd (runfiles path) for embeddedSchemaTemplate +// to run instead of migrating in this process, which under -race takes ~20s. +const schemaToolEnv = "BEADS_TEST_EMBEDDED_SCHEMA_TOOL" + +// embeddedSchemaTemplate creates one embedded Dolt database per test process +// with every schema migration applied and nothing else (embeddeddolt.Open on +// an empty directory, as bd init's own open does), under testTempRoot, and +// returns its database directory. +func embeddedSchemaTemplate() (string, error) { + schemaTemplateOnce.Do(func() { + beadsDir, err := testTempDir("bd-schema-template-*") + if err != nil { + schemaTemplateErr = fmt.Errorf("create schema template dir: %w", err) + return + } + if err := migrateSchemaTemplate(beadsDir); err != nil { + schemaTemplateErr = fmt.Errorf("migrate schema template in %s: %w", beadsDir, err) + return + } + schemaTemplateDir = filepath.Join(beadsDir, "embeddeddolt", schemaTemplateDatabase) + }) + return schemaTemplateDir, schemaTemplateErr +} + +func migrateSchemaTemplate(beadsDir string) error { + if os.Getenv(schemaToolEnv) == "" { + store, err := embeddeddolt.Open(context.Background(), beadsDir, schemaTemplateDatabase, "main") + if err != nil { + return err + } + return store.Close() + } + tool, err := bazeltest.RunfileEnv(schemaToolEnv) + if err != nil { + return err + } + out, err := exec.Command(tool, "--dir", beadsDir, "--database", schemaTemplateDatabase).CombinedOutput() + if err != nil { + return fmt.Errorf("%s: %w\n%s", tool, err, out) + } + return nil +} + +// seededInitDatabase returns the embedded database name `bd init --quiet +// ` creates, and false when bdInit should not seed it: an argument +// it does not know (it may pick another mode or location), no --prefix (the +// name then comes from the directory), or BEADS_TEST_FRESH_INIT=1. +func seededInitDatabase(extraArgs []string) (string, bool) { + if os.Getenv(freshInitEnv) == "1" { + return "", false + } + var prefix, database string + for i := 0; i < len(extraArgs); i++ { + switch extraArgs[i] { + case "--prefix", "--database": + if i+1 == len(extraArgs) { + return "", false + } + if extraArgs[i] == "--prefix" { + prefix = extraArgs[i+1] + } else { + database = extraArgs[i+1] + } + i++ + case "--quiet", "--skip-hooks", "--skip-agents", "--non-interactive": + default: + return "", false + } + } + if prefix == "" { + return "", false + } + if database != "" { + return database, true + } + return dbNameFromPrefix(normalizeIssuePrefix(prefix)), true +} + +// seedEmbeddedSchema copies the schema template into dir's .beads as database +// before bd init runs there. bd init then opens a database whose migrations +// are all applied, so it skips the migration chain (one Dolt commit per +// migration: most of an init's cost under -race) and does everything else +// itself: identity, issue prefix, config, hooks, agent files, the git commit. +// Seeded workspaces share the template's migration commits as their history. +func seedEmbeddedSchema(t *testing.T, dir, database string) { + t.Helper() + tmpl, err := embeddedSchemaTemplate() + if err != nil { + t.Fatalf("%v", err) + } + dst := filepath.Join(dir, ".beads", "embeddeddolt", database) + if err := copyTree(tmpl, dst); err != nil { + t.Fatalf("seed embedded schema %s into %s: %v", tmpl, dst, err) + } +} + +// requireSeededEmbeddedInit fails unless bd init kept the seeded database as +// the workspace's embedded database. +func requireSeededEmbeddedInit(t *testing.T, beadsDir, database string) { + t.Helper() + cfg, err := configfile.Load(beadsDir) + if err != nil || cfg == nil { + t.Fatalf("bd init on a seeded schema: load %s/metadata.json: cfg=%v err=%v", beadsDir, cfg, err) + } + if cfg.DoltMode != configfile.DoltModeEmbedded || cfg.DoltDatabase != database { + t.Fatalf("bd init on a seeded schema chose mode %q database %q, want embedded %q; teach seededInitDatabase this case or run with %s=1", + cfg.DoltMode, cfg.DoltDatabase, database, freshInitEnv) + } +} + +// copyTree copies src's tree to dst, keeping file and directory modes. +func copyTree(src, dst string) error { + return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + info, err := d.Info() + if err != nil { + return err + } + switch { + case d.IsDir(): + if err := os.MkdirAll(target, 0o700); err != nil { + return err + } + return os.Chmod(target, info.Mode().Perm()|0o700) + case d.Type().IsRegular(): + b, err := os.ReadFile(path) + if err != nil { + return err + } + if err := os.WriteFile(target, b, info.Mode().Perm()); err != nil { + return err + } + return os.Chmod(target, info.Mode().Perm()) + default: + return fmt.Errorf("unexpected file type %s at %s", d.Type(), path) + } + }) +} + +func TestSeededInitDatabase(t *testing.T) { + t.Setenv(freshInitEnv, "") + for _, tt := range []struct { + args []string + want string + ok bool + }{ + {[]string{"--prefix", "gc"}, "gc", true}, + {[]string{"--prefix", "my-proj", "--skip-hooks", "--skip-agents"}, "my_proj", true}, + {[]string{"--prefix", "a.b"}, "a_b", true}, + {[]string{"--database", "shared_db", "--prefix", "alpha"}, "shared_db", true}, + {nil, "", false}, + {[]string{"--prefix"}, "", false}, + {[]string{"--prefix", "gc", "--server"}, "", false}, + {[]string{"--prefix", "gc", "--role", "contributor"}, "", false}, + } { + got, ok := seededInitDatabase(tt.args) + if got != tt.want || ok != tt.ok { + t.Errorf("seededInitDatabase(%q) = %q, %v; want %q, %v", tt.args, got, ok, tt.want, tt.ok) + } + } + t.Setenv(freshInitEnv, "1") + if _, ok := seededInitDatabase([]string{"--prefix", "gc"}); ok { + t.Errorf("seededInitDatabase seeds with %s=1", freshInitEnv) + } +} + +func TestCopyTreeKeepsModes(t *testing.T) { + src, dst := t.TempDir(), filepath.Join(t.TempDir(), "copy") + if err := os.MkdirAll(filepath.Join(src, "noms"), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "noms", "manifest"), []byte("m"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "hook"), []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + if err := copyTree(src, dst); err != nil { + t.Fatalf("copyTree: %v", err) + } + if b, err := os.ReadFile(filepath.Join(dst, "noms", "manifest")); err != nil || string(b) != "m" { + t.Fatalf("copied manifest = %q, %v", b, err) + } + for path, want := range map[string]os.FileMode{ + filepath.Join(dst, "noms"): 0o750, + filepath.Join(dst, "noms", "manifest"): 0o600, + filepath.Join(dst, "hook"): 0o755, + } { + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != want { + t.Errorf("%s mode = %v, want %v", path, info.Mode().Perm(), want) + } + } +} diff --git a/cmd/bd/init_embedded_test.go b/cmd/bd/init_embedded_test.go index 0c4963000..891985cd4 100644 --- a/cmd/bd/init_embedded_test.go +++ b/cmd/bd/init_embedded_test.go @@ -189,13 +189,22 @@ func bdRunWithFlockRetry(t *testing.T, bd, dir string, args ...string) ([]byte, // bdInit creates a temp dir with a git repo, runs bd init --quiet with the // given extra args, and returns (dir, beadsDir, combined output). -// Fatals if bd init fails. +// Fatals if bd init fails. bd init runs on a schema already migrated from a +// per-process template (seedEmbeddedSchema) when seededInitDatabase knows the +// arguments. func bdInit(t *testing.T, bd string, extraArgs ...string) (dir, beadsDir string, out string) { t.Helper() dir = t.TempDir() initGitRepoAt(t, dir) + database, seeded := seededInitDatabase(extraArgs) + if seeded { + seedEmbeddedSchema(t, dir, database) + } out = runBDInit(t, bd, dir, extraArgs...) beadsDir = filepath.Join(dir, ".beads") + if seeded { + requireSeededEmbeddedInit(t, beadsDir, database) + } return } @@ -1597,12 +1606,23 @@ func TestEmbeddedInitRoleRouting(t *testing.T) { } } -func TestEmbeddedInitSelectedExcludeRouting(t *testing.T) { +// The selected-exclude routing cases are two top-level tests, not one: each +// case is a full embedded init under -race and they cannot run in parallel +// (t.Setenv), so one function held a whole Bazel shard for ~2 minutes. +func TestEmbeddedInitSelectedExcludeRoutingStealth(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "stealth_decoy", "stealth_invalid", "quiet_stealth") +} + +func TestEmbeddedInitSelectedExcludeRoutingFork(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "fork_decoy", "fork_invalid", "quiet_fork", "fork_auto_invalid") +} + +func testEmbeddedInitSelectedExcludeRouting(t *testing.T, names ...string) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") } bd := buildEmbeddedBD(t) - for _, name := range []string{"stealth_decoy", "stealth_invalid", "fork_decoy", "fork_invalid", "quiet_stealth", "quiet_fork", "fork_auto_invalid"} { + for _, name := range names { t.Run(name, func(t *testing.T) { target, decoy, home := newInitRoleFixture(t) global := filepath.Join(home, ".gitconfig") diff --git a/internal/storage/embeddeddolt/cmd/BUILD.bazel b/internal/storage/embeddeddolt/cmd/BUILD.bazel index 8a9136e0b..840daec1e 100644 --- a/internal/storage/embeddeddolt/cmd/BUILD.bazel +++ b/internal/storage/embeddeddolt/cmd/BUILD.bazel @@ -22,6 +22,24 @@ go_binary( visibility = ["//:__subpackages__"], ) +# The same tool built without the race detector whatever the configuration, +# for //cmd/bd:bd_embedded_test's schema template: run once per shard, it +# migrates an empty database in seconds instead of the ~20s it takes under +# --config=embedded's race. cgo-only, as :cmd. +go_binary( + name = "cmd_norace", + testonly = True, + out = "cmd_norace/schema", + embed = [":cmd_lib"], + race = "off", + tags = ["cgo-only"], + target_compatible_with = select({ + "//tools/bazel:pure": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + visibility = ["//cmd/bd:__pkg__"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/scripts/ci/_embedded_shard_manifest_lib.py b/scripts/ci/_embedded_shard_manifest_lib.py index 2f234d77f..ac80a8db8 100644 --- a/scripts/ci/_embedded_shard_manifest_lib.py +++ b/scripts/ci/_embedded_shard_manifest_lib.py @@ -8,8 +8,8 @@ machinery, generalized so both embedded generators (and any future one) share one implementation instead of two independently-maintained copies. The per-suite scripts own discovery (the test-name regex/glob), the duration-file path and its "seconds_per_init_fallback" semantics, and the manifest header -text; this module owns everything suite-agnostic: pack(), incremental_pack(), -split_blocks()/write_block(), and check_coverage(). +text; this module owns everything suite-agnostic: pack(), pack_concurrent(), +incremental_pack(), split_blocks()/write_block(), and check_coverage(). See gen_proxied_shard_manifest.py's module docstring for the full rationale behind incremental --write (merge-conflict avoidance across unrelated PRs) @@ -32,6 +32,35 @@ def pack(costs, total): return shards, loads +def shard_wall(names, costs, serial, parallelism): + """Estimated wall time of one Go test process running `names`: + top-level tests that do not call t.Parallel (`serial`) run one after + another before the parallel ones, which then share `parallelism` + (-test.parallel) slots, so the parallel group takes at least its longest + test and at least its total spread over the slots.""" + ser = sum(costs[n] for n in names if n in serial) + par = [costs[n] for n in names if n not in serial] + return ser + max(max(par, default=0.0), sum(par) / parallelism) + + +def pack_concurrent(costs, total, serial, parallelism): + """Greedy bin-pack for a Go test process that runs tests concurrently: + longest first, each onto the shard whose estimated wall time + (shard_wall) grows least, ties to the lighter shard. Plain LPT (pack()) + sums durations, which for a -test.parallel binary overstates a shard of + several parallel tests and understates one holding a serial test. + Returns (shards, walls).""" + order = sorted(costs, key=lambda k: (-costs[k], k)) + shards = [[] for _ in range(total)] + walls = [0.0] * total + for name in order: + best = min(range(total), key=lambda i: ( + shard_wall(shards[i] + [name], costs, serial, parallelism), walls[i], i)) + shards[best].append(name) + walls[best] = shard_wall(shards[best], costs, serial, parallelism) + return shards, walls + + def split_blocks(text): """Split a manifest file's text into its per-total_shards blocks. diff --git a/scripts/ci/embedded_cmd_test_durations.json b/scripts/ci/embedded_cmd_test_durations.json index aa935db5c..e5fcfe221 100644 --- a/scripts/ci/embedded_cmd_test_durations.json +++ b/scripts/ci/embedded_cmd_test_durations.json @@ -1,315 +1,290 @@ { "_comment": [ - "Measured locally with `bazel --nohome_rc test --config=embedded", - "//cmd/bd:bd_embedded_test --local_test_jobs=8` alongside the storage-tier", - "and conformance targets below (all four run in the same invocation, the", - "same way bazel.yml's bazel-embedded tier runs its shards concurrently on", - "one host) -- race, `-test.parallel=4`, tag filter `embedded`, run with the", - "real committed 40-shard manifest (.github/scripts/embedded-cmd-test-shards.txt)", - "so every one of the 242 TestEmbedded* functions that existed at capture time", - "ran exactly once (local sandboxed execution only; no remote executor or", - "cache flags were configured for this measurement run).", + "Measured on rbe-west, the lane's own remote executors (instance oss), on", + "2026-10-09: two `bazel test --config=embedded --nocache_test_results", + "//cmd/bd:bd_embedded_test` runs (race, `-test.parallel=4`, 50 shards), so", + "every top-level TestEmbedded* function ran exactly once per run. Each value", + "is the mean of the two runs (the two split", + "TestEmbeddedInitSelectedExcludeRouting{Stealth,Fork} functions ran only in", + "the second). The runs were taken after bdInit started seeding each", + "workspace's embedded database from a per-process schema template", + "(cmd/bd/embedded_init_template_test.go) and the bd subprocesses stopped", + "sleeping 1s at exit (GORACE=atexit_sleep_ms=0 on the target): the sum of", + "test-seconds fell from ~20.2k to ~10.3k on the same executors. Individual", + "tests varied up to ~2x between the two runs with executor load; the sum", + "moved ~4%. The previous capture was local sandboxed execution and", + "correlated only ~0.4 with rbe-west shard times.", "", "Each value is that top-level test's own from Bazel's", - "generated test.xml, NOT the enclosing . For this Go test", - "binary / Bazel version, was found empirically to be a", - "wall-clock-since-binary-start timestamp (it does not isolate a concurrently", - "-run top-level test's own duration under -test.parallel=4), whereas", - " matched the test binary's own `--- PASS: TestX (Ys)` line", - "exactly. Do not switch this capture to without", - "re-verifying that relationship for the toolchain in use at the time.", + "generated test.xml, NOT the enclosing : is a wall-clock-since-binary-start timestamp that does not isolate a", + "concurrently-run top-level test's own duration under -test.parallel=4,", + "while matches the test binary's own `--- PASS: TestX (Ys)`", + "line. Go adds a parallel test's elapsed time to its duration before it", + "waits for a parallel slot, so slot wait is not counted.", "", - "F1 review correction (S1): an earlier pass of this file explained four", - "outliers (TestEmbeddedUpdateConcurrent, TestEmbeddedCloseConcurrent,", - "TestEmbeddedInitA, TestEmbeddedEpicCloseEligibleReason) as a", - "\"queueing-delay\" artifact of Go's t.Parallel() clock starting before a", - "test's parallel slot is granted, and shipped corrected-down weights on", - "that theory. That explanation was WRONG and has been retracted: Go's", - "testing package (testing.go's (*T).Parallel) adds the elapsed time to", - "t.duration BEFORE blocking on the parallel-slot barrier, so queued-for-a-", - "slot wait time is never counted in a test's own reported duration in the", - "first place. The original large numbers were genuine CPU-bound running", - "time under real resource contention (dozens of Bazel shards running", - "concurrently on one host, each internally up to -test.parallel=4) -- this", - "capture re-measures every test under exactly that condition (all 54", - "cmd+storage+conformance shards running together, not solo) instead of", - "mixing solo-measured corrections into a contended-capture baseline.", + "The generator packs these with a model of the shard process", + "(_embedded_shard_manifest_lib.py's pack_concurrent): functions without a", + "top-level t.Parallel() run one after another, before the parallel ones,", + "which share -test.parallel=4 slots.", "", - "Two of those four outliers (TestEmbeddedUpdateConcurrent,", - "TestEmbeddedCloseConcurrent) are synthetic many-worker concurrency stress", - "tests whose cost is dominated by serialized `bd` subprocess invocations", - "behind the embedded engine's single-writer lock, not by -test.parallel", - "contention; their recorded values below are post-fix estimates (measured", - "value x 0.4) after cutting issuesPerWorker from 5 to 2 in both tests (see", - "their doc comments in update_embedded_test.go / close_embedded_test.go) --", - "pending a fresh capture to confirm. TestEmbeddedEpicCloseEligibleReason", - "was split into TestEmbeddedEpicCloseEligibleReasonFlag (3 subtests) and", - "TestEmbeddedEpicCloseEligibleReasonOutput (2 subtests); their recorded", - "values are the measured combined total apportioned 3:2 by subtest count,", - "also pending a fresh capture. TestEmbeddedInitA needed no further action:", - "at ~196s under this consistent methodology it is close to but under the", - "~200s single-test isolation threshold.", + "`seconds_per_init_fallback` is sum(durations) / sum(inits) over every test", + "here (inits: discover_inits_cost()'s t.Run( count proxy), used only for a", + "brand-new test with no entry yet.", "", - "Every test measuring at or above ~200s under this consistent load is", - "pinned alone on its own shard in the generated manifest below (see that", - "file's header); LPT bin-packing does this automatically as long as the", - "weight is accurate, since no combination of lighter tests sums close to", - "a ~200s+ single item without the packer preferring to isolate it.", - "", - "`seconds_per_init_fallback` is the weighted average seconds-per-init over", - "every measured test in this file (sum of all durations / sum of all inits,", - "where inits is discover_inits_cost()'s t.Run( count proxy), used only for a", - "brand-new test with no entry yet (see", - "scripts/ci/gen_embedded_cmd_shard_manifest.py's discover_inits_cost()).", - "", - "As with the proxied tier's duration file, these are a relative weight for", - "LPT bin-packing, not an absolute SLA: within-shard -test.parallel=4", - "overlap, farm noisy-neighbor effects, and future code changes all mean a", - "shard's total packed cost is a planning estimate, not a guarantee." + "These are a relative weight for packing, not an absolute SLA." ], - "seconds_per_init_fallback": 19.4828, + "seconds_per_init_fallback": 9.5837, "tests": { - "TestEmbeddedAdminBlocked": 20.33, - "TestEmbeddedAdminCompactReadOnlyAllowed": 21.99, - "TestEmbeddedAdminCompactTier2Honesty": 25.2, - "TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber": 28.23, - "TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll": 23.63, - "TestEmbeddedAutoImportJSONLConcurrentWriter": 34.41, - "TestEmbeddedAutoImportJSONLNoExplicitCommit": 64.27, - "TestEmbeddedAutoImportJSONLSkipsNonEmpty": 27.53, - "TestEmbeddedBackup": 162.68, - "TestEmbeddedBackupConcurrent": 45.4, - "TestEmbeddedBlocked": 35.34, - "TestEmbeddedBlockedConcurrent": 51.71, - "TestEmbeddedBlockedLabelFilters": 66.12, - "TestEmbeddedBootstrap": 133.62, - "TestEmbeddedBootstrapConcurrent": 23.4, - "TestEmbeddedBranch": 128.64, - "TestEmbeddedBranchConcurrent": 36.84, - "TestEmbeddedChangeDirOverridesInheritedBeadsDir": 62.33, - "TestEmbeddedChildren": 86.18, - "TestEmbeddedChildrenConcurrent": 104.16, - "TestEmbeddedClaimPools": 78.4, - "TestEmbeddedCloseAlreadyClosed": 179.15, - "TestEmbeddedCloseBasic": 158.81, - "TestEmbeddedCloseConcurrent": 129.7, - "TestEmbeddedCloseGuardsAndEpics": 105.86, - "TestEmbeddedComments": 30.42, - "TestEmbeddedCommentsCLI": 89.97, - "TestEmbeddedCommentsCLIConcurrent": 38.85, - "TestEmbeddedCompact": 166.61, - "TestEmbeddedCompactConcurrent": 28.81, - "TestEmbeddedCompactMisdatedChild": 33.71, - "TestEmbeddedConfig": 83.45, - "TestEmbeddedConfigConcurrent": 96.47, - "TestEmbeddedConfigNoWorkspaceStoreFreeCommands": 6.09, - "TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout": 0.35, - "TestEmbeddedContext": 68.26, - "TestEmbeddedContextConcurrent": 22.8, - "TestEmbeddedContributorCreate": 166.08, - "TestEmbeddedCount": 127.95, - "TestEmbeddedCountConcurrent": 122.44, - "TestEmbeddedCountIncludeEphemeral": 86.31, - "TestEmbeddedCountIncludeInfra": 80.89, - "TestEmbeddedCreateCommitPending": 41.89, - "TestEmbeddedCreateConcurrent": 39.83, - "TestEmbeddedCreateCrossRepo": 44.64, - "TestEmbeddedCreateCrossRepoDryRunWithParent": 62.24, - "TestEmbeddedCreateCrossRepoUninit": 45.69, - "TestEmbeddedCreateCrossRepoWithParent": 80.27, - "TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget": 48.57, - "TestEmbeddedCreateDryRunDoesNotMigrate": 28.63, - "TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget": 24.56, - "TestEmbeddedCreateFormCommitsLabelOnlyCreate": 31.46, - "TestEmbeddedCreateGraphAndBulk": 114.91, - "TestEmbeddedCreateJSONEchoesSubSecondTimestamps": 32.3, - "TestEmbeddedCreateRepoFromNonBeadsCwd": 23.45, - "TestEmbeddedCreateRepoProxiedTargetError": 45.4, - "TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected": 45.93, - "TestEmbeddedCreateRepoRelativeUninitRefused": 24.13, - "TestEmbeddedCreateScalarAndRelationships": 191.35, - "TestEmbeddedCreateStorageClassBasic": 111.77, - "TestEmbeddedCreateStorageClassBatchAndMisc": 163.68, - "TestEmbeddedCreateStorageClassConflicts": 93.79, - "TestEmbeddedCreateWithGitRemote": 25.17, - "TestEmbeddedDefer": 92.03, - "TestEmbeddedDeferAutoWake": 79.42, - "TestEmbeddedDeferConcurrent": 121.28, - "TestEmbeddedDelete": 122.43, - "TestEmbeddedDeleteJSONDependencyErrorContract": 58.17, - "TestEmbeddedDepA": 175.73, - "TestEmbeddedDepAddWispSource": 58.14, - "TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead": 132.31, - "TestEmbeddedDepB": 176.75, - "TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate": 38.51, - "TestEmbeddedDepConcurrent": 150.67, - "TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge": 66.98, - "TestEmbeddedDepNoCycleCheck": 88.95, - "TestEmbeddedDiff": 56.44, - "TestEmbeddedDiffConcurrent": 68.69, - "TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead": 174.02, - "TestEmbeddedDoltBlockedAndCommit": 59.12, - "TestEmbeddedDoltConcurrent": 68.7, - "TestEmbeddedDoltRemoteAndPush": 136.04, - "TestEmbeddedDuplicate": 72.32, - "TestEmbeddedDuplicateConcurrent": 69.38, - "TestEmbeddedDuplicates": 173.68, - "TestEmbeddedDuplicatesConcurrent": 59.24, - "TestEmbeddedEpicCloseEligible": 149.47, - "TestEmbeddedEpicCloseEligibleReasonFlag": 169.79, - "TestEmbeddedEpicCloseEligibleReasonOutput": 113.2, - "TestEmbeddedEpicConcurrent": 142.99, - "TestEmbeddedEpicStatus": 95.11, - "TestEmbeddedExplicitHierarchicalParentEdges": 65.84, - "TestEmbeddedExportConcurrent": 44.93, - "TestEmbeddedExportFlags": 130.38, - "TestEmbeddedExportFormats": 115.36, - "TestEmbeddedExternalCapabilityReadyAfterShip": 63.55, - "TestEmbeddedExternalMutationPolicy": 116.06, - "TestEmbeddedFederationConcurrent": 31.34, - "TestEmbeddedFederationPeers": 146.53, - "TestEmbeddedFederationStatus": 133.93, - "TestEmbeddedFindDuplicates": 124.62, - "TestEmbeddedFindDuplicatesConcurrent": 70.27, - "TestEmbeddedFlatten": 161.6, - "TestEmbeddedFlattenConcurrent": 32.13, - "TestEmbeddedGC": 185.65, - "TestEmbeddedGCConcurrent": 40.17, - "TestEmbeddedGate": 181.85, - "TestEmbeddedGateConcurrent": 66.94, - "TestEmbeddedGateCreate": 123.16, - "TestEmbeddedGateListScoped": 38.87, - "TestEmbeddedGetDependencies": 59.85, - "TestEmbeddedGraph": 81.16, - "TestEmbeddedGraphConcurrent": 88.99, - "TestEmbeddedHistory": 109.52, - "TestEmbeddedHistoryConcurrent": 90.07, - "TestEmbeddedHooks": 27.53, - "TestEmbeddedHooksConcurrent": 23.45, - "TestEmbeddedHuman": 97.66, - "TestEmbeddedHumanConcurrent": 30.64, - "TestEmbeddedImportClosedChildWhoseParentWasDeleted": 21.26, - "TestEmbeddedImportConcurrent": 31.4, - "TestEmbeddedImportFromSources": 124.38, - "TestEmbeddedImportPromotedWispRoundtrip": 137.68, - "TestEmbeddedImportUpsertAndPrefixSync": 111.37, - "TestEmbeddedInfo": 41.38, - "TestEmbeddedInfoConcurrent": 39.05, - "TestEmbeddedInfoSuppressGitHooks": 33.35, - "TestEmbeddedInitA": 196.25, - "TestEmbeddedInitArtifactRouting": 22.67, - "TestEmbeddedInitB": 145.22, - "TestEmbeddedInitC": 23.58, - "TestEmbeddedInitConcurrent": 47.95, - "TestEmbeddedInitExplicitBeadsDirIsAuthoritative": 53.4, - "TestEmbeddedInitGitBootstrapRouting": 158.28, - "TestEmbeddedInitHooksRouting": 22.24, - "TestEmbeddedInitRoleRouting": 125.08, - "TestEmbeddedInitSelectedExcludeRouting": 153.06, - "TestEmbeddedKV": 48.98, - "TestEmbeddedKVConcurrent": 108.72, - "TestEmbeddedLabelAddRemove": 138.68, - "TestEmbeddedLabelConcurrent": 84.16, - "TestEmbeddedLabelEditReports": 138.9, - "TestEmbeddedLabelPropagateRename": 120.78, - "TestEmbeddedLint": 82.71, - "TestEmbeddedLintConcurrent": 75.72, - "TestEmbeddedListComments": 51.81, - "TestEmbeddedListConcurrent": 33.41, - "TestEmbeddedListFiltering": 239.7, - "TestEmbeddedListSearchAndOutput": 147.31, - "TestEmbeddedMaxRowsDepTreeAndDuplicates": 97.69, - "TestEmbeddedMaxRowsDoctorAndConfig": 90.88, - "TestEmbeddedMaxRowsGraph": 159.26, - "TestEmbeddedMaxRowsListBasic": 107.6, - "TestEmbeddedMaxRowsListLimitInteraction": 192.0, - "TestEmbeddedMaxRowsReady": 168.09, - "TestEmbeddedMaxRowsReadyMerge": 78.54, - "TestEmbeddedMemory": 130.17, - "TestEmbeddedMemoryConcurrent": 95.43, - "TestEmbeddedMigrate": 155.64, - "TestEmbeddedMigrateConcurrent": 32.07, - "TestEmbeddedOnboard": 25.75, - "TestEmbeddedOnboardConcurrent": 22.91, + "TestEmbeddedAdminBlocked": 17.82, + "TestEmbeddedAdminCompactReadOnlyAllowed": 2.5, + "TestEmbeddedAdminCompactTier2Honesty": 17.71, + "TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber": 18.77, + "TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll": 11.63, + "TestEmbeddedAutoImportJSONLConcurrentWriter": 29.04, + "TestEmbeddedAutoImportJSONLNoExplicitCommit": 33.58, + "TestEmbeddedAutoImportJSONLSkipsNonEmpty": 8.05, + "TestEmbeddedBackup": 37.73, + "TestEmbeddedBackupConcurrent": 11.05, + "TestEmbeddedBlocked": 14.0, + "TestEmbeddedBlockedConcurrent": 28.59, + "TestEmbeddedBlockedLabelFilters": 21.34, + "TestEmbeddedBootstrap": 80.85, + "TestEmbeddedBootstrapConcurrent": 9.56, + "TestEmbeddedBranch": 25.91, + "TestEmbeddedBranchConcurrent": 27.48, + "TestEmbeddedChangeDirOverridesInheritedBeadsDir": 17.26, + "TestEmbeddedChildren": 71.69, + "TestEmbeddedChildrenConcurrent": 37.41, + "TestEmbeddedClaimPools": 19.91, + "TestEmbeddedCloseAlreadyClosed": 84.22, + "TestEmbeddedCloseBasic": 72.03, + "TestEmbeddedCloseConcurrent": 84.28, + "TestEmbeddedCloseGuardsAndEpics": 58.54, + "TestEmbeddedComments": 10.35, + "TestEmbeddedCommentsCLI": 50.71, + "TestEmbeddedCommentsCLIConcurrent": 29.98, + "TestEmbeddedCompact": 41.59, + "TestEmbeddedCompactConcurrent": 23.3, + "TestEmbeddedCompactMisdatedChild": 12.83, + "TestEmbeddedConfig": 44.69, + "TestEmbeddedConfigConcurrent": 82.94, + "TestEmbeddedConfigNoWorkspaceStoreFreeCommands": 0.95, + "TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout": 0.32, + "TestEmbeddedContext": 24.87, + "TestEmbeddedContextConcurrent": 13.71, + "TestEmbeddedContributorCreate": 87.03, + "TestEmbeddedCount": 59.63, + "TestEmbeddedCountConcurrent": 44.3, + "TestEmbeddedCountIncludeEphemeral": 24.09, + "TestEmbeddedCountIncludeInfra": 53.12, + "TestEmbeddedCreateCommitPending": 12.9, + "TestEmbeddedCreateConcurrent": 29.6, + "TestEmbeddedCreateCrossRepo": 43.14, + "TestEmbeddedCreateCrossRepoDryRunWithParent": 46.25, + "TestEmbeddedCreateCrossRepoUninit": 35.37, + "TestEmbeddedCreateCrossRepoWithParent": 41.61, + "TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget": 17.89, + "TestEmbeddedCreateDryRunDoesNotMigrate": 15.28, + "TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget": 3.09, + "TestEmbeddedCreateFormCommitsLabelOnlyCreate": 3.25, + "TestEmbeddedCreateGraphAndBulk": 16.55, + "TestEmbeddedCreateJSONEchoesSubSecondTimestamps": 8.31, + "TestEmbeddedCreateRepoFromNonBeadsCwd": 12.31, + "TestEmbeddedCreateRepoProxiedTargetError": 21.86, + "TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected": 56.2, + "TestEmbeddedCreateRepoRelativeUninitRefused": 4.97, + "TestEmbeddedCreateScalarAndRelationships": 57.97, + "TestEmbeddedCreateStorageClassBasic": 11.92, + "TestEmbeddedCreateStorageClassBatchAndMisc": 45.95, + "TestEmbeddedCreateStorageClassConflicts": 41.81, + "TestEmbeddedCreateWithGitRemote": 14.38, + "TestEmbeddedDefer": 38.48, + "TestEmbeddedDeferAutoWake": 48.29, + "TestEmbeddedDeferConcurrent": 63.55, + "TestEmbeddedDelete": 81.25, + "TestEmbeddedDeleteJSONDependencyErrorContract": 42.77, + "TestEmbeddedDepA": 95.56, + "TestEmbeddedDepAddWispSource": 28.54, + "TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead": 50.02, + "TestEmbeddedDepB": 94.49, + "TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate": 30.7, + "TestEmbeddedDepConcurrent": 75.13, + "TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge": 66.92, + "TestEmbeddedDepNoCycleCheck": 58.88, + "TestEmbeddedDiff": 41.65, + "TestEmbeddedDiffConcurrent": 49.97, + "TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead": 44.86, + "TestEmbeddedDoltBlockedAndCommit": 23.54, + "TestEmbeddedDoltConcurrent": 35.01, + "TestEmbeddedDoltRemoteAndPush": 18.79, + "TestEmbeddedDuplicate": 50.93, + "TestEmbeddedDuplicateConcurrent": 43.52, + "TestEmbeddedDuplicates": 67.03, + "TestEmbeddedDuplicatesConcurrent": 41.37, + "TestEmbeddedEpicCloseEligible": 69.55, + "TestEmbeddedEpicCloseEligibleReasonFlag": 29.56, + "TestEmbeddedEpicCloseEligibleReasonOutput": 41.91, + "TestEmbeddedEpicConcurrent": 71.22, + "TestEmbeddedEpicStatus": 44.06, + "TestEmbeddedExplicitHierarchicalParentEdges": 42.56, + "TestEmbeddedExportConcurrent": 21.66, + "TestEmbeddedExportFlags": 33.17, + "TestEmbeddedExportFormats": 40.58, + "TestEmbeddedExternalCapabilityReadyAfterShip": 37.72, + "TestEmbeddedExternalMutationPolicy": 64.3, + "TestEmbeddedFederationConcurrent": 25.94, + "TestEmbeddedFederationPeers": 38.08, + "TestEmbeddedFederationStatus": 27.84, + "TestEmbeddedFindDuplicates": 46.3, + "TestEmbeddedFindDuplicatesConcurrent": 14.36, + "TestEmbeddedFlatten": 31.29, + "TestEmbeddedFlattenConcurrent": 23.66, + "TestEmbeddedGC": 73.04, + "TestEmbeddedGCConcurrent": 21.52, + "TestEmbeddedGCConditionalMatcherDecode": 47.53, + "TestEmbeddedGate": 82.88, + "TestEmbeddedGateConcurrent": 49.5, + "TestEmbeddedGateCreate": 106.42, + "TestEmbeddedGateInheritsDownParentChain": 16.67, + "TestEmbeddedGateListScoped": 24.41, + "TestEmbeddedGatedRendering": 35.48, + "TestEmbeddedGetDependencies": 32.27, + "TestEmbeddedGraph": 61.4, + "TestEmbeddedGraphConcurrent": 21.38, + "TestEmbeddedHistory": 51.11, + "TestEmbeddedHistoryConcurrent": 77.78, + "TestEmbeddedHooks": 12.87, + "TestEmbeddedHooksConcurrent": 9.27, + "TestEmbeddedHuman": 66.13, + "TestEmbeddedHumanConcurrent": 9.72, + "TestEmbeddedIfRevisionReopenGuardDirect": 25.66, + "TestEmbeddedImportClosedChildWhoseParentWasDeleted": 27.28, + "TestEmbeddedImportConcurrent": 9.73, + "TestEmbeddedImportFromSources": 34.15, + "TestEmbeddedImportPromotedWispRoundtrip": 47.53, + "TestEmbeddedImportUpsertAndPrefixSync": 30.33, + "TestEmbeddedInfo": 17.36, + "TestEmbeddedInfoConcurrent": 33.86, + "TestEmbeddedInfoSuppressGitHooks": 9.64, + "TestEmbeddedInitA": 130.47, + "TestEmbeddedInitArtifactRouting": 20.34, + "TestEmbeddedInitB": 74.62, + "TestEmbeddedInitC": 32.69, + "TestEmbeddedInitConcurrent": 41.45, + "TestEmbeddedInitExplicitBeadsDirIsAuthoritative": 52.89, + "TestEmbeddedInitGitBootstrapRouting": 85.16, + "TestEmbeddedInitHooksRouting": 20.7, + "TestEmbeddedInitRoleRouting": 87.59, + "TestEmbeddedInitSelectedExcludeRoutingFork": 40.08, + "TestEmbeddedInitSelectedExcludeRoutingStealth": 65.73, + "TestEmbeddedKV": 12.62, + "TestEmbeddedKVConcurrent": 89.53, + "TestEmbeddedLabelAddRemove": 75.79, + "TestEmbeddedLabelConcurrent": 73.39, + "TestEmbeddedLabelEditReports": 68.91, + "TestEmbeddedLabelPropagateRename": 62.05, + "TestEmbeddedLint": 48.55, + "TestEmbeddedLintConcurrent": 44.72, + "TestEmbeddedListComments": 33.84, + "TestEmbeddedListConcurrent": 18.84, + "TestEmbeddedListFiltering": 87.75, + "TestEmbeddedListSearchAndOutput": 69.79, + "TestEmbeddedMaxRowsDepTreeAndDuplicates": 61.67, + "TestEmbeddedMaxRowsDoctorAndConfig": 30.57, + "TestEmbeddedMaxRowsGraph": 57.33, + "TestEmbeddedMaxRowsListBasic": 72.66, + "TestEmbeddedMaxRowsListLimitInteraction": 97.48, + "TestEmbeddedMaxRowsReady": 37.02, + "TestEmbeddedMaxRowsReadyMerge": 38.06, + "TestEmbeddedMemory": 57.23, + "TestEmbeddedMemoryConcurrent": 61.83, + "TestEmbeddedMemoryCorpusBudget": 17.38, + "TestEmbeddedMigrate": 34.92, + "TestEmbeddedMigrateConcurrent": 12.19, + "TestEmbeddedOnboard": 10.21, + "TestEmbeddedOnboardConcurrent": 19.27, "TestEmbeddedOpen_EmptyDatabaseRejected": 0.0, - "TestEmbeddedOrphans": 40.45, - "TestEmbeddedOrphansConcurrent": 65.99, - "TestEmbeddedPreviewDoesNotConsumeVersionMarker": 29.99, - "TestEmbeddedPrime": 32.69, - "TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget": 4.13, - "TestEmbeddedPrimeChangeDirPrimesTarget": 6.83, - "TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget": 5.69, - "TestEmbeddedPrimeConcurrent": 55.99, - "TestEmbeddedPromote": 24.64, - "TestEmbeddedPromoteCLI": 64.87, - "TestEmbeddedPromoteCLIConcurrent": 67.99, - "TestEmbeddedPruneDryRunAndFilters": 155.68, - "TestEmbeddedPruneSafetyAndScope": 163.85, - "TestEmbeddedPurgeBasics": 89.31, - "TestEmbeddedPurgeConcurrent": 38.42, - "TestEmbeddedPurgeFiltered": 131.05, - "TestEmbeddedPurgeWispsPlaneRetention": 119.54, - "TestEmbeddedQuery": 69.55, - "TestEmbeddedQueryConcurrent": 79.23, - "TestEmbeddedQuick": 108.95, - "TestEmbeddedQuickConcurrent": 52.9, - "TestEmbeddedQuickstart": 21.7, - "TestEmbeddedQuickstartConcurrent": 24.95, - "TestEmbeddedQuietFlagSuppressesSuccessOutput": 37.91, - "TestEmbeddedReady": 148.56, - "TestEmbeddedReadyClaimConcurrent": 37.22, - "TestEmbeddedReadyConcurrent": 39.82, - "TestEmbeddedRenamePrefixBasic": 111.8, - "TestEmbeddedRenamePrefixConcurrent": 30.67, - "TestEmbeddedRenamePrefixEdgeCases": 81.14, - "TestEmbeddedReopen": 105.91, - "TestEmbeddedReopenConcurrent": 112.14, - "TestEmbeddedRestore": 137.93, + "TestEmbeddedOrphans": 25.62, + "TestEmbeddedOrphansConcurrent": 31.05, + "TestEmbeddedPreviewDoesNotConsumeVersionMarker": 13.76, + "TestEmbeddedPrime": 17.08, + "TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget": 0.87, + "TestEmbeddedPrimeChangeDirPrimesTarget": 0.95, + "TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget": 0.85, + "TestEmbeddedPrimeConcurrent": 22.81, + "TestEmbeddedPromote": 13.89, + "TestEmbeddedPromoteCLI": 48.09, + "TestEmbeddedPromoteCLIConcurrent": 52.29, + "TestEmbeddedPruneDryRunAndFilters": 41.08, + "TestEmbeddedPruneSafetyAndScope": 63.09, + "TestEmbeddedPurgeBasics": 33.45, + "TestEmbeddedPurgeConcurrent": 28.04, + "TestEmbeddedPurgeFiltered": 40.47, + "TestEmbeddedPurgeWispsPlaneRetention": 76.18, + "TestEmbeddedQuery": 40.0, + "TestEmbeddedQueryConcurrent": 45.96, + "TestEmbeddedQuick": 51.42, + "TestEmbeddedQuickConcurrent": 61.4, + "TestEmbeddedQuickstart": 9.7, + "TestEmbeddedQuickstartConcurrent": 15.87, + "TestEmbeddedQuietFlagSuppressesSuccessOutput": 31.54, + "TestEmbeddedReadOnlyCommandsSpawnNoGit": 33.33, + "TestEmbeddedReady": 71.28, + "TestEmbeddedReadyClaimConcurrent": 26.71, + "TestEmbeddedReadyConcurrent": 32.41, + "TestEmbeddedReclaimRunsTheLeaseReclaimerRole": 26.56, + "TestEmbeddedRenamePrefixBasic": 46.34, + "TestEmbeddedRenamePrefixConcurrent": 18.87, + "TestEmbeddedRenamePrefixEdgeCases": 31.27, + "TestEmbeddedReopen": 59.7, + "TestEmbeddedReopenConcurrent": 76.86, + "TestEmbeddedRestore": 35.94, "TestEmbeddedRestoreConcurrent": 0.0, - "TestEmbeddedRoutedMutatingSiblingAssignTagDep": 156.88, - "TestEmbeddedRoutedMutatingSiblingDeleteClose": 101.7, - "TestEmbeddedRoutedSiblingWritesCommitTargetHead": 191.64, - "TestEmbeddedSearch": 118.78, - "TestEmbeddedSearchConcurrent": 98.05, - "TestEmbeddedSetup": 52.64, - "TestEmbeddedSetupConcurrent": 22.64, - "TestEmbeddedShip": 71.62, - "TestEmbeddedShipConcurrent": 101.01, - "TestEmbeddedShowBasicsAndJSON": 118.13, - "TestEmbeddedShowConcurrent": 150.5, - "TestEmbeddedShowDetailAndCurrent": 135.15, - "TestEmbeddedShowUnresolvableDeps": 59.52, - "TestEmbeddedSquashWispMoleculeClearsRootEphemeral": 23.37, - "TestEmbeddedStale": 90.45, - "TestEmbeddedStaleConcurrent": 79.59, - "TestEmbeddedStaleLabelFilters": 84.1, - "TestEmbeddedState": 66.22, - "TestEmbeddedStateConcurrent": 45.78, - "TestEmbeddedStatus": 47.37, - "TestEmbeddedStatusConcurrent": 79.04, - "TestEmbeddedSupersede": 76.14, - "TestEmbeddedSupersedeConcurrent": 72.31, - "TestEmbeddedSwarm": 172.12, - "TestEmbeddedSwarmConcurrent": 71.02, - "TestEmbeddedTodo": 74.74, - "TestEmbeddedTodoConcurrent": 30.6, - "TestEmbeddedTypes": 46.4, - "TestEmbeddedTypesAgreeWithCreate": 109.4, - "TestEmbeddedTypesConcurrent": 29.96, - "TestEmbeddedUnclaim": 130.59, - "TestEmbeddedUndefer": 122.4, - "TestEmbeddedUndeferConcurrent": 121.9, - "TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead": 28.26, - "TestEmbeddedUpdateConcurrent": 180.79, - "TestEmbeddedUpdateFields": 142.3, - "TestEmbeddedUpdateLifecycle": 150.21, - "TestEmbeddedUpdateRoutedStoreCommitsTargetHead": 51.5, - "TestEmbeddedVCCommitVariantsAndMerge": 105.56, - "TestEmbeddedVCConcurrent": 56.84, - "TestEmbeddedVCStatusAndCommit": 126.19, - "TestEmbeddedVersion": 1.4, - "TestEmbeddedVersionConcurrent": 1.48, - "TestEmbeddedWhere": 55.16, - "TestEmbeddedWhereConcurrent": 27.1 + "TestEmbeddedRoutedMutatingSiblingAssignTagDep": 105.3, + "TestEmbeddedRoutedMutatingSiblingDeleteClose": 74.67, + "TestEmbeddedRoutedSiblingWritesCommitTargetHead": 83.5, + "TestEmbeddedSearch": 65.75, + "TestEmbeddedSearchConcurrent": 61.47, + "TestEmbeddedSetup": 19.29, + "TestEmbeddedSetupConcurrent": 16.88, + "TestEmbeddedShip": 47.53, + "TestEmbeddedShipConcurrent": 68.2, + "TestEmbeddedShowBasicsAndJSON": 55.39, + "TestEmbeddedShowConcurrent": 77.92, + "TestEmbeddedShowDetailAndCurrent": 82.5, + "TestEmbeddedShowUnresolvableDeps": 56.8, + "TestEmbeddedSquashWispMoleculeClearsRootEphemeral": 26.27, + "TestEmbeddedStale": 63.39, + "TestEmbeddedStaleConcurrent": 46.3, + "TestEmbeddedStaleLabelFilters": 26.04, + "TestEmbeddedState": 55.29, + "TestEmbeddedStateConcurrent": 42.2, + "TestEmbeddedStatus": 29.52, + "TestEmbeddedStatusConcurrent": 53.34, + "TestEmbeddedSupersede": 46.47, + "TestEmbeddedSupersedeConcurrent": 70.75, + "TestEmbeddedSwarm": 97.32, + "TestEmbeddedSwarmConcurrent": 49.07, + "TestEmbeddedTodo": 49.91, + "TestEmbeddedTodoConcurrent": 19.93, + "TestEmbeddedTypes": 22.43, + "TestEmbeddedTypesAgreeWithCreate": 37.24, + "TestEmbeddedTypesConcurrent": 20.36, + "TestEmbeddedUnclaim": 78.85, + "TestEmbeddedUndefer": 52.25, + "TestEmbeddedUndeferConcurrent": 58.19, + "TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead": 14.69, + "TestEmbeddedUpdateConcurrent": 105.17, + "TestEmbeddedUpdateFields": 77.51, + "TestEmbeddedUpdateLifecycle": 73.02, + "TestEmbeddedUpdateRoutedStoreCommitsTargetHead": 45.79, + "TestEmbeddedVCCommitVariantsAndMerge": 14.78, + "TestEmbeddedVCConcurrent": 41.36, + "TestEmbeddedVCStatusAndCommit": 44.06, + "TestEmbeddedVersion": 0.35, + "TestEmbeddedVersionConcurrent": 0.53, + "TestEmbeddedWhere": 11.55, + "TestEmbeddedWhereConcurrent": 8.26 } } diff --git a/scripts/ci/gen_embedded_cmd_shard_manifest.py b/scripts/ci/gen_embedded_cmd_shard_manifest.py index a93255eff..4087338e6 100755 --- a/scripts/ci/gen_embedded_cmd_shard_manifest.py +++ b/scripts/ci/gen_embedded_cmd_shard_manifest.py @@ -18,7 +18,9 @@ docstring for the full rationale this one shares): scripts/ci/embedded_cmd_test_durations.json (see that file's header for provenance). A function missing from that file — a test added since it was last captured — falls back to its inits cost times the file's - "seconds_per_init_fallback" ratio. + "seconds_per_init_fallback" ratio. A from-scratch pack models the shard + process (pack_concurrent): functions without a top-level t.Parallel() + run one after another, the rest share -test.parallel=4 slots. This file's existing 20-shard block (hand-assigned, round-robin; see its own header) is the FROZEN block PR Risk's and main.yml's legacy fork/push jobs @@ -47,12 +49,15 @@ import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from _embedded_shard_manifest_lib import ( # noqa: E402 - check_coverage, incremental_pack, pack, read_assignments, write_block, + check_coverage, incremental_pack, pack, pack_concurrent, read_assignments, write_block, ) default_manifest_path = '.github/scripts/embedded-cmd-test-shards.txt' durations_filename = 'embedded_cmd_test_durations.json' default_total_shards = 20 +# .bazelrc's test:embedded --test_arg=-test.parallel=4: how many t.Parallel +# top-level tests one shard's process runs at once (--weights=duration). +test_parallelism = 4 func_re = re.compile(r'^func (TestEmbedded[A-Za-z0-9_]+)\(') @@ -78,6 +83,27 @@ def discover_inits_cost(): return costs +def discover_serial(): + """Return the TestEmbedded* functions whose body does not call + t.Parallel() directly: Go runs them one at a time, before the parallel + ones (a test that reaches t.Parallel only through a helper counts as + serial, which overestimates its shard).""" + serial = set() + for path in glob.glob('cmd/bd/*_embedded_test.go'): + with open(path) as fh: + lines = fh.readlines() + cur, parallel = None, False + for ln in lines + ['func \n']: + if ln.startswith('func '): + if cur and not parallel: + serial.add(cur) + m = func_re.match(ln) + cur, parallel = (m.group(1) if m else None), False + elif cur and ln.rstrip('\n') == '\tt.Parallel()': + parallel = True + return serial + + def duration_cost(inits): """Return {test_name: seconds}, from the committed duration capture, falling back to inits * seconds_per_init_fallback for an undated test.""" @@ -115,9 +141,11 @@ def render(total, shards, weights): else: out.append(f'# {total}-shard split for the Bazel-only embedded-Dolt cmd tier') out.append('# (bazel-embedded in .github/workflows/bazel.yml), bin-packed') - out.append('# longest-processing-time-first by measured wall-time from') + out.append('# longest-first by measured wall-time from') out.append(f'# scripts/ci/{durations_filename} (see that file for provenance') - out.append('# and its "relative weight, not absolute SLA" caveat).') + out.append('# and its "relative weight, not absolute SLA" caveat) onto the shard') + out.append('# whose estimated wall time grows least: serial tests (no top-level') + out.append(f'# t.Parallel) add up, parallel ones share -test.parallel={test_parallelism}.') out.append(f'# Regenerate with scripts/ci/gen_embedded_cmd_shard_manifest.py {total}') out.append('# --weights=duration --write after adding, splitting or removing') out.append('# TestEmbedded* functions in cmd/bd/*_embedded_test.go. --write is') @@ -180,6 +208,8 @@ def main(): existing = None if args.repack else read_assignments(args.manifest, args.total_shards) if existing: shards, loads = incremental_pack(costs, args.total_shards, existing) + elif args.weights == 'duration': + shards, loads = pack_concurrent(costs, args.total_shards, discover_serial(), test_parallelism) else: shards, loads = pack(costs, args.total_shards) out = render(args.total_shards, shards, args.weights) diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index a5853126f..52ff1b463 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -806,7 +806,12 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { want := map[string]target{ "//cmd/bd:bd_embedded_test": { []string{"$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", "-test.timeout=19m"}, - map[string]string{"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)"}, + map[string]string{ + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + // bdInit's schema template and the race runtime's exit sleep: + // neither selects or skips a test. + "BEADS_TEST_EMBEDDED_SCHEMA_TOOL": "$(rlocationpath //internal/storage/embeddeddolt/cmd:cmd_norace)", "GORACE": "atexit_sleep_ms=0", + }, }, "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": { []string{"$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)", "-test.timeout=19m"}, diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go index c0c61c9aa..134fbf905 100644 --- a/scripts/shard_manifest_test.go +++ b/scripts/shard_manifest_test.go @@ -43,6 +43,35 @@ func TestCmdEmbeddedShardManifestGeneratorNotStale(t *testing.T) { } } +// TestPackConcurrentModelsSerialAndParallelTests pins +// _embedded_shard_manifest_lib.py's pack_concurrent: a serial test (no +// top-level t.Parallel) holds its shard alone for its whole duration, while +// parallel tests share -test.parallel slots, so four 30s parallel tests fit +// beside a 90s one in the same wall time. Plain LPT would put the serial +// test with a parallel one and split the rest evenly by summed duration. +func TestPackConcurrentModelsSerialAndParallelTests(t *testing.T) { + python := requireHostTool(t, "python3") + root := sourceRepoRoot(t) + const script = ` +import sys +sys.path.insert(0, 'scripts/ci') +from _embedded_shard_manifest_lib import pack_concurrent +costs = {'serial': 100, 'a': 90, 'b': 80, 'c': 30, 'd': 30, 'e': 30, 'f': 30} +shards, walls = pack_concurrent(costs, 2, {'serial'}, 4) +print(sorted(map(sorted, shards)), walls) +` + cmd := exec.Command(python, "-c", script) + cmd.Dir = root + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("pack_concurrent: %v\n%s", err, out) + } + const want = "[['a', 'b', 'c', 'd', 'e', 'f'], ['serial']] [100.0, 90]\n" + if string(out) != want { + t.Errorf("pack_concurrent = %q, want %q", out, want) + } +} + // TestStorageEmbeddedShardManifestGeneratorNotStale mirrors // TestCmdEmbeddedShardManifestGeneratorNotStale above for the storage tier's // Bazel-only 20-shard block; see that test's doc comment. From 0d865460b19acd21e3a0924f237fd81079dd0256 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 20:30:52 +0000 Subject: [PATCH 021/151] ci(bazel): split the slow embedded cmd tests and run the cmd block as 2x50 shards (#7438) //cmd/bd:bd_embedded_test was the embedded lane's floor: at Bazel's 50-shard cap per rule, each shard's wall time was its slowest top-level test, and ~40 tests ran 65-130s on rbe-west. - 34 slow TestEmbedded* functions are split into 89 smaller top-level functions. Subtests and table cases are moved verbatim: no assertion, flag or data changes, the same t.Run set per file, the same skip guard and t.Parallel. Subtests that read state earlier ones left stay together, in order; each new function repeats the shared setup it needs. - The manifest's cmd/bd block is now 100 shards, run by two targets: bd_embedded_test (shards 1-50) and the new bd_embedded_part2_test (51-100), through go_test_manifest_shard.sh --shard-offset/--shard-total. check_shard_coverage.py takes SHARDS@OFFSET/TOTAL and fails unless the targets' ranges tile the block exactly once, so every test runs in exactly one shard of one target; the policy tests read both rules' ranges from cmd/bd/BUILD.bazel. - Durations re-measured on rbe-west with the split tests and repacked. The frozen legacy 20-shard block drops the renamed names (no workflow reads it; new names hash-distribute as before). Not split: the *Concurrent stress tests (their bd subprocesses queue on the workspace's single writer lock: TestEmbeddedUpdateConcurrent is ~100 serialized bd commands), TestEmbeddedPurgeWispsPlaneRetention (one stateful sequence), and the MaxRowsList* tests (setup is 21 sequential creates every subtest reads). rbe-west, both targets, --nocache_test_results: slowest shard 138-163s -> 109s (TestEmbeddedUpdateConcurrent alone; every other shard <=86s), median 80-84s -> 40s, shard-seconds 4.1-4.4k -> 4.2k, critical path 213-306s -> 113s. Bead: ga-vnycm2.17 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/embedded-cmd-test-shards.txt | 635 ++++++++++-------- .github/workflows/bazel.yml | 28 +- BUILD.bazel | 3 +- cmd/bd/BUILD.bazel | 67 +- cmd/bd/autocommit_embedded_test.go | 75 ++- cmd/bd/bootstrap_embedded_test.go | 11 +- cmd/bd/children_embedded_test.go | 46 +- cmd/bd/close_embedded_test.go | 49 +- cmd/bd/contributor_embedded_test.go | 22 +- cmd/bd/delete_embedded_test.go | 40 +- cmd/bd/dep_embedded_test.go | 259 ++++--- cmd/bd/duplicates_embedded_test.go | 79 ++- cmd/bd/epic_embedded_test.go | 34 +- cmd/bd/gate_embedded_test.go | 156 +++-- cmd/bd/gc_embedded_test.go | 34 +- cmd/bd/human_embedded_test.go | 12 +- cmd/bd/init_embedded_test.go | 112 ++- cmd/bd/label_embedded_test.go | 65 +- cmd/bd/list_embedded_test.go | 59 +- cmd/bd/ready_embedded_test.go | 249 +++---- cmd/bd/search_embedded_test.go | 162 +++-- cmd/bd/show_embedded_test.go | 21 +- cmd/bd/swarm_embedded_test.go | 41 +- cmd/bd/unclaim_embedded_test.go | 12 +- cmd/bd/update_embedded_test.go | 46 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 8 +- scripts/ci/embedded_cmd_test_durations.json | 576 ++++++++-------- scripts/ci/gen_embedded_cmd_shard_manifest.py | 4 +- scripts/ci_workflow_test.go | 9 +- scripts/embedded_shard_count_test.go | 88 ++- scripts/pr_risk_bazel_coverage_test.go | 73 +- tools/bazel/check_shard_coverage.py | 64 +- tools/bazel/go_test_manifest_shard.sh | 35 +- 33 files changed, 2047 insertions(+), 1127 deletions(-) diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 486e904fe..e6987e1f4 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -10,15 +10,11 @@ 20 2 TestEmbeddedBackupConcurrent 20 3 TestEmbeddedBlocked 20 4 TestEmbeddedBlockedConcurrent -20 5 TestEmbeddedBootstrap 20 6 TestEmbeddedBootstrapConcurrent 20 7 TestEmbeddedBranch 20 8 TestEmbeddedBranchConcurrent -20 9 TestEmbeddedChildren 20 10 TestEmbeddedChildrenConcurrent -20 11 TestEmbeddedCloseBasic 20 11 TestEmbeddedCloseGuardsAndEpics -20 11 TestEmbeddedCloseAlreadyClosed 20 12 TestEmbeddedCloseConcurrent 20 13 TestEmbeddedComments 20 14 TestEmbeddedCommentsCLI @@ -31,7 +27,6 @@ 20 1 TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout 20 2 TestEmbeddedContext 20 3 TestEmbeddedContextConcurrent -20 4 TestEmbeddedContributorCreate 20 5 TestEmbeddedCount 20 6 TestEmbeddedCountConcurrent 20 7 TestEmbeddedCreateScalarAndRelationships @@ -44,9 +39,6 @@ 20 13 TestEmbeddedCreateWithGitRemote 20 14 TestEmbeddedDefer 20 15 TestEmbeddedDeferConcurrent -20 16 TestEmbeddedDelete -20 17 TestEmbeddedDepA -20 17 TestEmbeddedDepB 20 18 TestEmbeddedDepConcurrent 20 19 TestEmbeddedDiff 20 20 TestEmbeddedDiffConcurrent @@ -55,10 +47,8 @@ 20 2 TestEmbeddedDoltConcurrent 20 3 TestEmbeddedDuplicate 20 4 TestEmbeddedDuplicateConcurrent -20 5 TestEmbeddedDuplicates 20 6 TestEmbeddedDuplicatesConcurrent 20 7 TestEmbeddedEpicStatus -20 7 TestEmbeddedEpicCloseEligible 20 7 TestEmbeddedEpicCloseEligibleReasonFlag 20 7 TestEmbeddedEpicCloseEligibleReasonOutput 20 8 TestEmbeddedEpicConcurrent @@ -72,11 +62,8 @@ 20 14 TestEmbeddedFindDuplicatesConcurrent 20 15 TestEmbeddedFlatten 20 16 TestEmbeddedFlattenConcurrent -20 17 TestEmbeddedGC 20 18 TestEmbeddedGCConcurrent -20 19 TestEmbeddedGate 20 20 TestEmbeddedGateConcurrent -20 1 TestEmbeddedGateCreate 20 2 TestEmbeddedGetDependencies 20 3 TestEmbeddedGraph 20 4 TestEmbeddedGraphConcurrent @@ -84,27 +71,20 @@ 20 6 TestEmbeddedHistoryConcurrent 20 7 TestEmbeddedHooks 20 8 TestEmbeddedHooksConcurrent -20 9 TestEmbeddedHuman 20 10 TestEmbeddedHumanConcurrent 20 11 TestEmbeddedImportFromSources 20 11 TestEmbeddedImportUpsertAndPrefixSync 20 12 TestEmbeddedImportConcurrent 20 13 TestEmbeddedInfo 20 14 TestEmbeddedInfoConcurrent -20 15 TestEmbeddedInitA -20 15 TestEmbeddedInitB 20 15 TestEmbeddedInitC 20 16 TestEmbeddedInitConcurrent 20 17 TestEmbeddedKV 20 18 TestEmbeddedKVConcurrent -20 19 TestEmbeddedLabelAddRemove -20 19 TestEmbeddedLabelEditReports 20 19 TestEmbeddedLabelPropagateRename 20 20 TestEmbeddedLabelConcurrent 20 1 TestEmbeddedLint 20 2 TestEmbeddedLintConcurrent -20 3 TestEmbeddedListFiltering -20 3 TestEmbeddedListSearchAndOutput 20 4 TestEmbeddedListConcurrent 20 13 TestEmbeddedMaxRowsListBasic 20 13 TestEmbeddedMaxRowsListLimitInteraction @@ -137,7 +117,6 @@ 20 4 TestEmbeddedQuickConcurrent 20 5 TestEmbeddedQuickstart 20 6 TestEmbeddedQuickstartConcurrent -20 7 TestEmbeddedReady 20 8 TestEmbeddedReadyConcurrent 20 9 TestEmbeddedRenamePrefixBasic 20 9 TestEmbeddedRenamePrefixEdgeCases @@ -146,14 +125,12 @@ 20 12 TestEmbeddedReopenConcurrent 20 13 TestEmbeddedRestore 20 14 TestEmbeddedRestoreConcurrent -20 15 TestEmbeddedSearch 20 16 TestEmbeddedSearchConcurrent 20 17 TestEmbeddedSetup 20 18 TestEmbeddedSetupConcurrent 20 19 TestEmbeddedShip 20 20 TestEmbeddedShipConcurrent 20 1 TestEmbeddedShowBasicsAndJSON -20 1 TestEmbeddedShowDetailAndCurrent 20 2 TestEmbeddedShowConcurrent 20 3 TestEmbeddedStale 20 4 TestEmbeddedStaleConcurrent @@ -163,7 +140,6 @@ 20 8 TestEmbeddedStatusConcurrent 20 9 TestEmbeddedSupersede 20 10 TestEmbeddedSupersedeConcurrent -20 11 TestEmbeddedSwarm 20 12 TestEmbeddedSwarmConcurrent 20 13 TestEmbeddedTodo 20 14 TestEmbeddedTodoConcurrent @@ -171,8 +147,6 @@ 20 16 TestEmbeddedTypesConcurrent 20 17 TestEmbeddedUndefer 20 18 TestEmbeddedUndeferConcurrent -20 19 TestEmbeddedUpdateFields -20 19 TestEmbeddedUpdateLifecycle 20 20 TestEmbeddedUpdateConcurrent 20 1 TestEmbeddedVCStatusAndCommit 20 1 TestEmbeddedVCCommitVariantsAndMerge @@ -186,14 +160,14 @@ # # Format: # -# 50-shard split for the Bazel-only embedded-Dolt cmd tier +# 100-shard split for the Bazel-only embedded-Dolt cmd tier # (bazel-embedded in .github/workflows/bazel.yml), bin-packed # longest-first by measured wall-time from # scripts/ci/embedded_cmd_test_durations.json (see that file for provenance # and its "relative weight, not absolute SLA" caveat) onto the shard # whose estimated wall time grows least: serial tests (no top-level # t.Parallel) add up, parallel ones share -test.parallel=4. -# Regenerate with scripts/ci/gen_embedded_cmd_shard_manifest.py 50 +# Regenerate with scripts/ci/gen_embedded_cmd_shard_manifest.py 100 # --weights=duration --write after adding, splitting or removing # TestEmbedded* functions in cmd/bd/*_embedded_test.go. --write is # incremental: it keeps every already-assigned test on its current shard @@ -208,302 +182,407 @@ # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator or the durations file instead. -50 1 TestEmbeddedInitA +100 1 TestEmbeddedUpdateConcurrent -50 2 TestEmbeddedGateCreate +100 2 TestEmbeddedShowConcurrent -50 3 TestEmbeddedRoutedMutatingSiblingAssignTagDep +100 3 TestEmbeddedCloseConcurrent -50 4 TestEmbeddedUpdateConcurrent +100 4 TestEmbeddedEpicConcurrent -50 5 TestEmbeddedMaxRowsListLimitInteraction +100 5 TestEmbeddedConfigConcurrent -50 6 TestEmbeddedSwarm +100 6 TestEmbeddedListFilteringBasicAndLabels -50 7 TestEmbeddedDepA +100 7 TestEmbeddedDepConcurrent -50 8 TestEmbeddedDepB +100 8 TestEmbeddedMaxRowsListLimitInteraction -50 9 TestEmbeddedKVConcurrent +100 9 TestEmbeddedChildrenConcurrent -50 10 TestEmbeddedListFiltering +100 10 TestEmbeddedCreateScalarAndRelationships -50 11 TestEmbeddedInitRoleRouting +100 11 TestEmbeddedKVConcurrent -50 12 TestEmbeddedContributorCreate +100 12 TestEmbeddedGatedRendering -50 13 TestEmbeddedInitGitBootstrapRouting +100 13 TestEmbeddedListFilteringStatusAndHierarchy -50 14 TestEmbeddedCloseConcurrent +100 14 TestEmbeddedReopenConcurrent -50 15 TestEmbeddedCloseAlreadyClosed +100 15 TestEmbeddedPurgeWispsPlaneRetention -50 16 TestEmbeddedRoutedSiblingWritesCommitTargetHead +100 16 TestEmbeddedMemoryConcurrent -50 17 TestEmbeddedConfigConcurrent +100 17 TestEmbeddedReopen -50 18 TestEmbeddedGate +100 18 TestEmbeddedLabelPropagateRename -50 19 TestEmbeddedShowDetailAndCurrent +100 19 TestEmbeddedLabelConcurrent -50 20 TestEmbeddedDelete +100 20 TestEmbeddedInitRoleRoutingRetainedFork -50 21 TestEmbeddedBootstrap +100 21 TestEmbeddedMaxRowsReady -50 22 TestEmbeddedUnclaim +100 22 TestEmbeddedCount -50 23 TestEmbeddedBootstrapConcurrent -50 23 TestEmbeddedHooksConcurrent -50 23 TestEmbeddedShowConcurrent +100 23 TestEmbeddedCountConcurrent -50 24 TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll -50 24 TestEmbeddedBackupConcurrent -50 24 TestEmbeddedBlocked -50 24 TestEmbeddedComments -50 24 TestEmbeddedCompactMisdatedChild -50 24 TestEmbeddedContextConcurrent -50 24 TestEmbeddedCreateCommitPending -50 24 TestEmbeddedCreateRepoFromNonBeadsCwd -50 24 TestEmbeddedCreateStorageClassBasic -50 24 TestEmbeddedCreateWithGitRemote -50 24 TestEmbeddedFindDuplicatesConcurrent -50 24 TestEmbeddedHistoryConcurrent -50 24 TestEmbeddedHooks -50 24 TestEmbeddedHumanConcurrent -50 24 TestEmbeddedInfoSuppressGitHooks -50 24 TestEmbeddedMigrateConcurrent -50 24 TestEmbeddedOnboard -50 24 TestEmbeddedPromote -50 24 TestEmbeddedQuickstart -50 24 TestEmbeddedWhere +100 24 TestEmbeddedUndeferConcurrent -50 25 TestEmbeddedAdminCompactReadOnlyAllowed -50 25 TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber -50 25 TestEmbeddedConfigNoWorkspaceStoreFreeCommands -50 25 TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout -50 25 TestEmbeddedCreateGraphAndBulk -50 25 TestEmbeddedDoltRemoteAndPush -50 25 TestEmbeddedGateInheritsDownParentChain -50 25 TestEmbeddedListConcurrent -50 25 TestEmbeddedOnboardConcurrent -50 25 TestEmbeddedPrime -50 25 TestEmbeddedQuickstartConcurrent -50 25 TestEmbeddedRenamePrefixConcurrent -50 25 TestEmbeddedSetup -50 25 TestEmbeddedSetupConcurrent -50 25 TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead -50 25 TestEmbeddedUpdateFields -50 25 TestEmbeddedVCCommitVariantsAndMerge -50 25 TestEmbeddedVersion -50 25 TestEmbeddedVersionConcurrent +100 25 TestEmbeddedMaxRowsListBasic -50 26 TestEmbeddedAdminBlocked -50 26 TestEmbeddedBlockedLabelFilters -50 26 TestEmbeddedClaimPools -50 26 TestEmbeddedCompactConcurrent -50 26 TestEmbeddedExportConcurrent -50 26 TestEmbeddedGCConcurrent -50 26 TestEmbeddedGraphConcurrent -50 26 TestEmbeddedPrimeChangeDirPrimesTarget -50 26 TestEmbeddedPrimeConcurrent -50 26 TestEmbeddedReopenConcurrent -50 26 TestEmbeddedTodoConcurrent -50 26 TestEmbeddedTypes -50 26 TestEmbeddedTypesConcurrent +100 26 TestEmbeddedExternalMutationPolicy -50 27 TestEmbeddedBranch -50 27 TestEmbeddedContext -50 27 TestEmbeddedCountIncludeEphemeral -50 27 TestEmbeddedCreateFormCommitsLabelOnlyCreate -50 27 TestEmbeddedFederationConcurrent -50 27 TestEmbeddedGateListScoped -50 27 TestEmbeddedIfRevisionReopenGuardDirect -50 27 TestEmbeddedOrphans -50 27 TestEmbeddedPurgeWispsPlaneRetention -50 27 TestEmbeddedReclaimRunsTheLeaseReclaimerRole -50 27 TestEmbeddedStaleLabelFilters +100 27 TestEmbeddedShipConcurrent -50 28 TestEmbeddedAutoImportJSONLConcurrentWriter -50 28 TestEmbeddedAutoImportJSONLSkipsNonEmpty -50 28 TestEmbeddedBranchConcurrent -50 28 TestEmbeddedCreateConcurrent -50 28 TestEmbeddedEpicCloseEligibleReasonFlag -50 28 TestEmbeddedFederationStatus -50 28 TestEmbeddedLabelAddRemove -50 28 TestEmbeddedPurgeConcurrent -50 28 TestEmbeddedReadyClaimConcurrent -50 28 TestEmbeddedStatus +100 28 TestEmbeddedHistoryConcurrent -50 29 TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate -50 29 TestEmbeddedDepConcurrent -50 29 TestEmbeddedFlatten -50 29 TestEmbeddedGetDependencies -50 29 TestEmbeddedImportUpsertAndPrefixSync -50 29 TestEmbeddedInfo -50 29 TestEmbeddedMaxRowsDoctorAndConfig -50 29 TestEmbeddedOrphansConcurrent -50 29 TestEmbeddedRenamePrefixEdgeCases +100 29 TestEmbeddedEpicCloseEligibleReasonFlag -50 30 TestEmbeddedAutoImportJSONLNoExplicitCommit -50 30 TestEmbeddedCreateJSONEchoesSubSecondTimestamps -50 30 TestEmbeddedDepAddWispSource -50 30 TestEmbeddedExportFlags -50 30 TestEmbeddedListComments -50 30 TestEmbeddedPurgeBasics -50 30 TestEmbeddedReadOnlyCommandsSpawnNoGit -50 30 TestEmbeddedReadyConcurrent -50 30 TestEmbeddedRoutedMutatingSiblingDeleteClose +100 30 TestEmbeddedCloseGuardsAndEpics -50 31 TestEmbeddedCreateCrossRepoUninit -50 31 TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget -50 31 TestEmbeddedDoltBlockedAndCommit -50 31 TestEmbeddedDoltConcurrent -50 31 TestEmbeddedGatedRendering -50 31 TestEmbeddedImportFromSources -50 31 TestEmbeddedInfoConcurrent -50 31 TestEmbeddedInitB -50 31 TestEmbeddedMigrate +100 31 TestEmbeddedInitSelectedExcludeRoutingForkInvalidQuiet -50 32 TestEmbeddedBackup -50 32 TestEmbeddedExternalCapabilityReadyAfterShip -50 32 TestEmbeddedFederationPeers -50 32 TestEmbeddedLabelConcurrent -50 32 TestEmbeddedMaxRowsReadyMerge -50 32 TestEmbeddedPreviewDoesNotConsumeVersionMarker -50 32 TestEmbeddedRestore -50 32 TestEmbeddedTypesAgreeWithCreate +100 32 TestEmbeddedListSearchAndOutputFormats -50 33 TestEmbeddedBlockedConcurrent -50 33 TestEmbeddedDefer -50 33 TestEmbeddedGC -50 33 TestEmbeddedPruneDryRunAndFilters -50 33 TestEmbeddedPurgeFiltered -50 33 TestEmbeddedQuery -50 33 TestEmbeddedVCConcurrent -50 33 TestEmbeddedWhereConcurrent +100 33 TestEmbeddedBootstrapJSONLAndGitOrigin -50 34 TestEmbeddedCompact -50 34 TestEmbeddedCreateCrossRepoWithParent -50 34 TestEmbeddedCreateRepoRelativeUninitRefused -50 34 TestEmbeddedCreateStorageClassConflicts -50 34 TestEmbeddedDuplicatesConcurrent -50 34 TestEmbeddedEpicCloseEligibleReasonOutput -50 34 TestEmbeddedFlattenConcurrent -50 34 TestEmbeddedOpen_EmptyDatabaseRejected -50 34 TestEmbeddedRestoreConcurrent -50 34 TestEmbeddedUpdateLifecycle +100 34 TestEmbeddedUnclaimBatchAndRefusals -50 35 TestEmbeddedCreateCrossRepo -50 35 TestEmbeddedDeleteJSONDependencyErrorContract -50 35 TestEmbeddedDuplicateConcurrent -50 35 TestEmbeddedInitArtifactRouting -50 35 TestEmbeddedMaxRowsListBasic -50 35 TestEmbeddedStateConcurrent -50 35 TestEmbeddedVCStatusAndCommit +100 35 TestEmbeddedGateCreateBeadRename -50 36 TestEmbeddedAdminCompactTier2Honesty -50 36 TestEmbeddedCloseBasic -50 36 TestEmbeddedConfig -50 36 TestEmbeddedCountConcurrent -50 36 TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead -50 36 TestEmbeddedInitConcurrent -50 36 TestEmbeddedLintConcurrent -50 36 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget +100 36 TestEmbeddedGateConcurrent -50 37 TestEmbeddedChangeDirOverridesInheritedBeadsDir -50 37 TestEmbeddedChildren -50 37 TestEmbeddedChildrenConcurrent -50 37 TestEmbeddedCreateCrossRepoDryRunWithParent -50 37 TestEmbeddedCreateStorageClassBatchAndMisc -50 37 TestEmbeddedQueryConcurrent -50 37 TestEmbeddedStaleConcurrent +100 37 TestEmbeddedMaxRowsDepTreeAndDuplicates -50 38 TestEmbeddedCreateDryRunDoesNotMigrate -50 38 TestEmbeddedImportPromotedWispRoundtrip -50 38 TestEmbeddedMaxRowsReady -50 38 TestEmbeddedReady -50 38 TestEmbeddedRenamePrefixBasic -50 38 TestEmbeddedShip -50 38 TestEmbeddedSupersede +100 38 TestEmbeddedGraphConcurrent -50 39 TestEmbeddedDeferAutoWake -50 39 TestEmbeddedEpicConcurrent -50 39 TestEmbeddedGateConcurrent -50 39 TestEmbeddedInitC -50 39 TestEmbeddedKV -50 39 TestEmbeddedLint -50 39 TestEmbeddedPromoteCLI +100 39 TestEmbeddedMaxRowsGraph -50 40 TestEmbeddedCommentsCLI -50 40 TestEmbeddedCommentsCLIConcurrent -50 40 TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead -50 40 TestEmbeddedDiffConcurrent -50 40 TestEmbeddedImportConcurrent -50 40 TestEmbeddedSupersedeConcurrent -50 40 TestEmbeddedTodo +100 40 TestEmbeddedUpdateLifecycleClaimParentCommit -50 41 TestEmbeddedHistory -50 41 TestEmbeddedInitSelectedExcludeRoutingFork -50 41 TestEmbeddedListSearchAndOutput -50 41 TestEmbeddedPromoteCLIConcurrent -50 41 TestEmbeddedQuick -50 41 TestEmbeddedUndefer +100 41 TestEmbeddedDuplicateConcurrent -50 42 TestEmbeddedCountIncludeInfra -50 42 TestEmbeddedDuplicate -50 42 TestEmbeddedEpicCloseEligible -50 42 TestEmbeddedInitExplicitBeadsDirIsAuthoritative -50 42 TestEmbeddedQuietFlagSuppressesSuccessOutput -50 42 TestEmbeddedStatusConcurrent +100 42 TestEmbeddedInitAForkAutoContributor -50 43 TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected -50 43 TestEmbeddedImportClosedChildWhoseParentWasDeleted -50 43 TestEmbeddedLabelEditReports -50 43 TestEmbeddedShowBasicsAndJSON -50 43 TestEmbeddedState -50 43 TestEmbeddedSwarmConcurrent +100 43 TestEmbeddedInitExplicitBeadsDirIsAuthoritative -50 44 TestEmbeddedFindDuplicates -50 44 TestEmbeddedMaxRowsGraph -50 44 TestEmbeddedMemory -50 44 TestEmbeddedShipConcurrent -50 44 TestEmbeddedShowUnresolvableDeps -50 44 TestEmbeddedSquashWispMoleculeClearsRootEphemeral +100 44 TestEmbeddedStaleConcurrent -50 45 TestEmbeddedCloseGuardsAndEpics -50 45 TestEmbeddedCreateRepoProxiedTargetError -50 45 TestEmbeddedCreateScalarAndRelationships -50 45 TestEmbeddedDuplicates -50 45 TestEmbeddedUndeferConcurrent -50 45 TestEmbeddedUpdateRoutedStoreCommitsTargetHead +100 45 TestEmbeddedGraph -50 46 TestEmbeddedCount -50 46 TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge -50 46 TestEmbeddedDepNoCycleCheck -50 46 TestEmbeddedEpicStatus -50 46 TestEmbeddedInitHooksRouting -50 46 TestEmbeddedReopen +100 46 TestEmbeddedCloseAlreadyClosedClaimNextAndMolecule -50 47 TestEmbeddedDiff -50 47 TestEmbeddedGraph -50 47 TestEmbeddedHuman -50 47 TestEmbeddedMemoryCorpusBudget -50 47 TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget -50 47 TestEmbeddedQuickConcurrent -50 47 TestEmbeddedSearchConcurrent +100 47 TestEmbeddedListSearchAndOutputSearchAndSort -50 48 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget -50 48 TestEmbeddedExportFormats -50 48 TestEmbeddedLabelPropagateRename -50 48 TestEmbeddedMaxRowsDepTreeAndDuplicates -50 48 TestEmbeddedMemoryConcurrent -50 48 TestEmbeddedSearch +100 48 TestEmbeddedDeferConcurrent -50 49 TestEmbeddedExplicitHierarchicalParentEdges -50 49 TestEmbeddedInitSelectedExcludeRoutingStealth +100 49 TestEmbeddedInitAPushConsentAndRemoteBootstrap -50 50 TestEmbeddedDeferConcurrent -50 50 TestEmbeddedExternalMutationPolicy -50 50 TestEmbeddedGCConditionalMatcherDecode -50 50 TestEmbeddedPruneSafetyAndScope -50 50 TestEmbeddedStale +100 50 TestEmbeddedDiffConcurrent + +100 51 TestEmbeddedInitGitBootstrapRoutingFreshQuiet + +100 52 TestEmbeddedSwarmConcurrent + +100 53 TestEmbeddedInitSelectedExcludeRoutingStealthDecoyQuiet + +100 54 TestEmbeddedSearchConcurrent + +100 55 TestEmbeddedUndefer + +100 56 TestEmbeddedDepNoCycleCheck + +100 57 TestEmbeddedQueryConcurrent + +100 58 TestEmbeddedDepAAddRejectionsAndBulk + +100 59 TestEmbeddedDeferAutoWake + +100 60 TestEmbeddedCommentsCLI + +100 61 TestEmbeddedUpdateRoutedStoreCommitsTargetHead + +100 62 TestEmbeddedReadyBlockersAndLabels + +100 63 TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead + +100 64 TestEmbeddedGateWaiterCheckLifecycle + +100 65 TestEmbeddedSupersede + +100 66 TestEmbeddedCloseBasicIsolatedStores + +100 67 TestEmbeddedAdminCompactReadOnlyAllowed +100 67 TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber +100 67 TestEmbeddedBootstrapConcurrent +100 67 TestEmbeddedCountIncludeInfra +100 67 TestEmbeddedCreateCommitPending +100 67 TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget +100 67 TestEmbeddedCreateRepoFromNonBeadsCwd +100 67 TestEmbeddedCreateRepoRelativeUninitRefused +100 67 TestEmbeddedHooks +100 67 TestEmbeddedOnboard +100 67 TestEmbeddedPrime +100 67 TestEmbeddedPromote +100 67 TestEmbeddedQuickstart +100 67 TestEmbeddedQuickstartConcurrent +100 67 TestEmbeddedWhere +100 67 TestEmbeddedWhereConcurrent + +100 68 TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll +100 68 TestEmbeddedBlocked +100 68 TestEmbeddedCompactConcurrent +100 68 TestEmbeddedContext +100 68 TestEmbeddedCreateJSONEchoesSubSecondTimestamps +100 68 TestEmbeddedDoltBlockedAndCommit +100 68 TestEmbeddedDoltConcurrent +100 68 TestEmbeddedInfo +100 68 TestEmbeddedInfoSuppressGitHooks +100 68 TestEmbeddedOrphans +100 68 TestEmbeddedRoutedMutatingSiblingDeleteCloseDelete +100 68 TestEmbeddedSetupConcurrent +100 68 TestEmbeddedTodoConcurrent +100 68 TestEmbeddedTypes + +100 69 TestEmbeddedCompact +100 69 TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout +100 69 TestEmbeddedCreateConcurrent +100 69 TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate +100 69 TestEmbeddedDuplicatesSharedFixture +100 69 TestEmbeddedGateListScoped +100 69 TestEmbeddedGetDependencies +100 69 TestEmbeddedInfoConcurrent +100 69 TestEmbeddedRenamePrefixConcurrent +100 69 TestEmbeddedRoutedSiblingWritesCommitTargetHeadNote +100 69 TestEmbeddedTypesConcurrent +100 69 TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead + +100 70 TestEmbeddedExportConcurrent +100 70 TestEmbeddedFlattenConcurrent +100 70 TestEmbeddedGCDryRunAndSkips +100 70 TestEmbeddedImportConcurrent +100 70 TestEmbeddedImportUpsertAndPrefixSync +100 70 TestEmbeddedLabelAddRemoveMultiAndDuplicate +100 70 TestEmbeddedMemory +100 70 TestEmbeddedMigrateConcurrent +100 70 TestEmbeddedPurgeConcurrent +100 70 TestEmbeddedRoutedMutatingSiblingAssignTagDepTag + +100 71 TestEmbeddedInitRoleRoutingExplicitDefault +100 71 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget + +100 72 TestEmbeddedContributorCreateMultiple +100 72 TestEmbeddedCreateWithGitRemote +100 72 TestEmbeddedFederationConcurrent +100 72 TestEmbeddedGateCreateRepoErrorsList +100 72 TestEmbeddedHumanConcurrent +100 72 TestEmbeddedImportPromotedWispRoundtrip +100 72 TestEmbeddedPrimeChangeDirPrimesTarget +100 72 TestEmbeddedRoutedMutatingSiblingDeleteCloseClose +100 72 TestEmbeddedRoutedSiblingWritesCommitTargetHeadReopen +100 72 TestEmbeddedUpdateFieldsMappingAndRefs + +100 73 TestEmbeddedCompactMisdatedChild +100 73 TestEmbeddedExportFlags +100 73 TestEmbeddedFederationPeers +100 73 TestEmbeddedGateCreateFlags +100 73 TestEmbeddedGateCreateReady +100 73 TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget +100 73 TestEmbeddedQuickConcurrent +100 73 TestEmbeddedRenamePrefixBasic +100 73 TestEmbeddedRoutedMutatingSiblingAssignTagDepDepAdd + +100 74 TestEmbeddedBlockedConcurrent +100 74 TestEmbeddedBranch +100 74 TestEmbeddedCountIncludeEphemeral +100 74 TestEmbeddedDepAddWispSource +100 74 TestEmbeddedMigrate +100 74 TestEmbeddedPrimeConcurrent +100 74 TestEmbeddedRestore +100 74 TestEmbeddedVCConcurrent + +100 75 TestEmbeddedCommentsCLIConcurrent +100 75 TestEmbeddedContextConcurrent +100 75 TestEmbeddedFlatten +100 75 TestEmbeddedInitC +100 75 TestEmbeddedListComments +100 75 TestEmbeddedOpen_EmptyDatabaseRejected +100 75 TestEmbeddedReclaimRunsTheLeaseReclaimerRole +100 75 TestEmbeddedRoutedSiblingWritesCommitTargetHeadComment +100 75 TestEmbeddedVCStatusAndCommit + +100 76 TestEmbeddedCloseAlreadyClosedContinue +100 76 TestEmbeddedCloseAlreadyClosedMultipleIDs +100 76 TestEmbeddedExportFormats +100 76 TestEmbeddedGCForceWithDecay +100 76 TestEmbeddedGCFullAndNoForcePrompts +100 76 TestEmbeddedInitAGitOriginRemote +100 76 TestEmbeddedLabelAddRemoveRemove +100 76 TestEmbeddedOnboardConcurrent + +100 77 TestEmbeddedClaimPools +100 77 TestEmbeddedConfigNoWorkspaceStoreFreeCommands +100 77 TestEmbeddedCreateStorageClassConflicts +100 77 TestEmbeddedExternalCapabilityReadyAfterShip +100 77 TestEmbeddedHumanListStatsRespondDismiss +100 77 TestEmbeddedMemoryCorpusBudget +100 77 TestEmbeddedOrphansConcurrent +100 77 TestEmbeddedStatus + +100 78 TestEmbeddedBranchConcurrent +100 78 TestEmbeddedComments +100 78 TestEmbeddedDepBRemove +100 78 TestEmbeddedLabelAddRemoveAdd +100 78 TestEmbeddedShip +100 78 TestEmbeddedUpdateFieldsNotesOverwrite +100 78 TestEmbeddedVCCommitVariantsAndMerge +100 78 TestEmbeddedVersionConcurrent + +100 79 TestEmbeddedCreateCrossRepoUninit +100 79 TestEmbeddedDeleteJSONDependencyErrorContract +100 79 TestEmbeddedEpicCloseEligibleNotEligible +100 79 TestEmbeddedInitABasic +100 79 TestEmbeddedInitBRemoteSchemaAndCloneFailure +100 79 TestEmbeddedRenamePrefixEdgeCases +100 79 TestEmbeddedUpdateFieldsNotesClearAndDefer + +100 80 TestEmbeddedCreateCrossRepoWithParent +100 80 TestEmbeddedCreateFormCommitsLabelOnlyCreate +100 80 TestEmbeddedCreateStorageClassBasic +100 80 TestEmbeddedRoutedMutatingSiblingAssignTagDepAssign +100 80 TestEmbeddedStatusConcurrent +100 80 TestEmbeddedSwarmCreate +100 80 TestEmbeddedSwarmValidate + +100 81 TestEmbeddedAdminCompactTier2Honesty +100 81 TestEmbeddedBackup +100 81 TestEmbeddedDeleteForceAndBatch +100 81 TestEmbeddedGateCreateBeadCheck +100 81 TestEmbeddedGateList +100 81 TestEmbeddedRestoreConcurrent +100 81 TestEmbeddedStaleLabelFilters +100 81 TestEmbeddedStateConcurrent + +100 82 TestEmbeddedCreateCrossRepoDryRunWithParent +100 82 TestEmbeddedDeleteBasic +100 82 TestEmbeddedDepBTreeAndCycles +100 82 TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead +100 82 TestEmbeddedHooksConcurrent +100 82 TestEmbeddedImportFromSources +100 82 TestEmbeddedStale + +100 83 TestEmbeddedAdminBlocked +100 83 TestEmbeddedAutoImportJSONLConcurrentWriter +100 83 TestEmbeddedInitBAutoCommitAndJSONLRemote +100 83 TestEmbeddedLabelEditReportsPrefix +100 83 TestEmbeddedMaxRowsDoctorAndConfig +100 83 TestEmbeddedPurgeFiltered +100 83 TestEmbeddedReadyConcurrent + +100 84 TestEmbeddedInitGitBootstrapRoutingDecoyInvalid +100 84 TestEmbeddedInitSelectedExcludeRoutingStealthInvalid + +100 85 TestEmbeddedBootstrapExistingDryRunInit +100 85 TestEmbeddedChildrenBasicAndEmpty +100 85 TestEmbeddedCreateGraphAndBulk +100 85 TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected +100 85 TestEmbeddedIfRevisionReopenGuardDirect +100 85 TestEmbeddedInitArtifactRouting + +100 86 TestEmbeddedExplicitHierarchicalParentEdges +100 86 TestEmbeddedInitHooksRouting + +100 87 TestEmbeddedCreateRepoProxiedTargetError +100 87 TestEmbeddedGateInheritsDownParentChain +100 87 TestEmbeddedReadyClaimConcurrent +100 87 TestEmbeddedState +100 87 TestEmbeddedSupersedeConcurrent +100 87 TestEmbeddedSwarmCreateForceAndErrors + +100 88 TestEmbeddedChangeDirOverridesInheritedBeadsDir +100 88 TestEmbeddedDepAAddAndList +100 88 TestEmbeddedKV +100 88 TestEmbeddedMaxRowsReadyMerge +100 88 TestEmbeddedPromoteCLI +100 88 TestEmbeddedUnclaimSuccess + +100 89 TestEmbeddedBackupConcurrent +100 89 TestEmbeddedCreateDryRunDoesNotMigrate +100 89 TestEmbeddedGateShowResolve +100 89 TestEmbeddedInitBStealthAndForceReinit +100 89 TestEmbeddedLabelEditReportsNoop +100 89 TestEmbeddedShowBasicsAndJSON + +100 90 TestEmbeddedAutoImportJSONLSkipsNonEmpty +100 90 TestEmbeddedConfig +100 90 TestEmbeddedFindDuplicatesConcurrent +100 90 TestEmbeddedLabelEditReportsListAndTemplate +100 90 TestEmbeddedLintConcurrent +100 90 TestEmbeddedUpdateLifecycleDefer +100 90 TestEmbeddedVersion + +100 91 TestEmbeddedDuplicate +100 91 TestEmbeddedPreviewDoesNotConsumeVersionMarker +100 91 TestEmbeddedPruneDryRunAndFilters +100 91 TestEmbeddedReadOnlyCommandsSpawnNoGit +100 91 TestEmbeddedShowUnresolvableDeps +100 91 TestEmbeddedTypesAgreeWithCreate + +100 92 TestEmbeddedAutoImportJSONLNoExplicitCommit +100 92 TestEmbeddedDefer +100 92 TestEmbeddedDepAAddTypesAndFlags +100 92 TestEmbeddedEpicCloseEligibleClosesEpics +100 92 TestEmbeddedSetup +100 92 TestEmbeddedSwarmStatusList + +100 93 TestEmbeddedCreateStorageClassBatchAndMisc +100 93 TestEmbeddedDeletePreviewAndGuards +100 93 TestEmbeddedDepBRemoveOutputAndRelatesTo +100 93 TestEmbeddedListConcurrent +100 93 TestEmbeddedPruneSafetyAndScope +100 93 TestEmbeddedShowDetailAndCurrentFlags + +100 94 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget +100 94 TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge +100 94 TestEmbeddedEpicCloseEligibleReasonOutput +100 94 TestEmbeddedFederationStatus +100 94 TestEmbeddedGCConcurrent +100 94 TestEmbeddedReadyDefaultJSONAndFlags + +100 95 TestEmbeddedChildrenJSONAndAllStatuses +100 95 TestEmbeddedFindDuplicates +100 95 TestEmbeddedImportClosedChildWhoseParentWasDeleted +100 95 TestEmbeddedInitConcurrent +100 95 TestEmbeddedQuick + +100 96 TestEmbeddedDiff +100 96 TestEmbeddedDuplicatesConcurrent +100 96 TestEmbeddedHistory +100 96 TestEmbeddedShowDetailAndCurrentHistoryAndCurrent +100 96 TestEmbeddedSquashWispMoleculeClearsRootEphemeral + +100 97 TestEmbeddedDuplicatesOwnWorkspaces +100 97 TestEmbeddedHumanPositionalFileAndGuards +100 97 TestEmbeddedPurgeBasics +100 97 TestEmbeddedQuietFlagSuppressesSuccessOutput +100 97 TestEmbeddedSearchStatusLimitSortDatesMetadata + +100 98 TestEmbeddedCloseBasicSharedStore +100 98 TestEmbeddedContributorCreateRoutesToPlanning +100 98 TestEmbeddedGCConditionalMatcherDecode +100 98 TestEmbeddedQuery +100 98 TestEmbeddedTodo + +100 99 TestEmbeddedBlockedLabelFilters +100 99 TestEmbeddedDoltRemoteAndPush +100 99 TestEmbeddedEpicStatus +100 99 TestEmbeddedInitSelectedExcludeRoutingForkDecoyAuto +100 99 TestEmbeddedSearchQueryStatusAssigneeLabels + +100 100 TestEmbeddedContributorCreateShowReadsRouted +100 100 TestEmbeddedCreateCrossRepo +100 100 TestEmbeddedInitGitBootstrapRoutingExistingExplicit +100 100 TestEmbeddedLint +100 100 TestEmbeddedPromoteCLIConcurrent diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index e434e70d3..9e24e688b 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1297,20 +1297,17 @@ jobs: # heaviest shard (scripts/ci/gen_embedded_storage_shard_manifest.py # --weights=duration --repack, on rbe-west measurements) stays near 110s, # under the ~150s per-shard budget with headroom for executor load; its - # heaviest single top-level test is ~90s. cmd (50) sits at Bazel's hard - # cap per sh_test-like rule ("Having more than 50 shards is indicative of - # poor test organization. Please reduce the number of shards." -- a - # native analysis-time error, not a style lint); on rbe-west its shards - # ran 83-209s (6.5k shard-seconds, 2026-10-09) until bdInit seeded each - # workspace from a per-process schema template and the race bd - # subprocesses stopped sleeping 1s at exit (~4.1-4.4k shard-seconds, - # slowest shard ~140-160s, its slowest single tests ~100-130s under - # load), so it, not storage or //internal/httpclient:httpclient_served_test, - # is still this lane's floor. Lowering it further needs per-test speedups - # in its slowest functions; more shards (splitting bd_embedded_test into - # multiple <=50-shard sh_test rules with shard-index-offset plumbing - # through tools/bazel/go_test_manifest_shard.sh and - # check_shard_coverage.py) would not move the slowest single test. On RBE the tier's wall time ~= that floor + # heaviest single top-level test is ~90s. cmd is a 100-shard block split + # over two targets, //cmd/bd:bd_embedded_test (shards 1-50) and + # //cmd/bd:bd_embedded_part2_test (51-100), because Bazel caps a rule at + # 50 shards ("Having more than 50 shards is indicative of poor test + # organization. Please reduce the number of shards." -- a native + # analysis-time error, not a style lint); the step below checks that + # the two ranges tile the block exactly once. After its slowest + # functions were split into smaller top-level tests, a cmd shard's wall + # time is its slowest test: ~45s median, and at most the *Concurrent + # stress tests' 65-145s (their bd subprocesses queue on one workspace's + # writer lock, so they cannot be split without changing what they test). On RBE the tier's wall time ~= that floor # + fixed cost (checkout, build, shard dispatch), not ~= # sum(durations)/shard_count the way a CPU-core-bound local sweep is -- # each shard is its own remote action and runs concurrently with every @@ -1397,7 +1394,8 @@ jobs: run: >- python3 tools/bazel/check_shard_coverage.py --bep "$RUNNER_TEMP/bazel-bep.json" - --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 50 + --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 50@0/100 + --suite //cmd/bd:bd_embedded_part2_test .github/scripts/embedded-test-shard.sh 50@50/100 --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 20 --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test diff --git a/BUILD.bazel b/BUILD.bazel index 69157bf54..134a91d1d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -125,7 +125,8 @@ exports_files( # PR Risk's manifest-sharded tiers' shard scripts and their committed # manifests (embedded-Dolt cmd and storage, proxied-server cmd, server-Dolt -# storage). The variants (//cmd/bd:bd_embedded_test, //cmd/bd:bd_proxied_test, +# storage). The variants (//cmd/bd:bd_embedded_test and +# //cmd/bd:bd_embedded_part2_test, //cmd/bd:bd_proxied_test, # //internal/storage/embeddeddolt:embeddeddolt_embedded_test, # //internal/storage/dolt:dolt_server_full_test) run them through # tools/bazel/go_test_manifest_shard.sh, so each Bazel shard runs exactly the diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 00cf8b088..d808cbd5d 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1396,17 +1396,22 @@ go_test( }), ) -# Runs 50 Bazel shards against the TestEmbedded* functions of -# *_embedded_test.go, assigned by the manifest's 50-shard block -# (duration-balanced; see scripts/ci/embedded_cmd_test_durations.json), -# against this configuration's bd_test with BEADS_TEST_EMBEDDED_DOLT=1 and -# the race bd (//cmd/bd:bd under --config=embedded) as the subprocess -# binary. This is a *different* shard count from PR Risk's legacy fork/push -# jobs, "Test (Embedded Dolt Cmd N/20)" (pr-risk.yml): those keep -# running 20 shards of their own, frozen hand-assigned block in the same -# manifest file — shard k here is not the same split as their shard k. It -# repeats bd_test's `env` (the wrapper cannot inherit it). shard_count must -# stay equal to this lane's manifest block's shard total. +# bd_embedded_test and bd_embedded_part2_test run the TestEmbedded* functions +# of *_embedded_test.go between them: the manifest's 100-shard block +# (duration-balanced; see scripts/ci/embedded_cmd_test_durations.json) is +# more than Bazel's 50-shard cap per rule, so bd_embedded_test runs its +# shards 1-50 and bd_embedded_part2_test its shards 51-100 +# (go_test_manifest_shard.sh --shard-offset/--shard-total; bazel.yml's +# check_shard_coverage.py fails unless the two ranges tile the block exactly +# once). Both run this configuration's bd_test with +# BEADS_TEST_EMBEDDED_DOLT=1 and the race bd (//cmd/bd:bd under +# --config=embedded) as the subprocess binary. This is a *different* shard +# count from PR Risk's legacy fork/push jobs, "Test (Embedded Dolt Cmd +# N/20)" (pr-risk.yml): those keep running 20 shards of their own, frozen +# hand-assigned block in the same manifest file — shard k here is not the +# same split as their shard k. Each repeats bd_test's `env` (the wrapper +# cannot inherit it). The two shard_counts must sum to --shard-total, the +# manifest block's shard total. # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, @@ -1422,6 +1427,8 @@ sh_test( timeout = "eternal", srcs = ["//tools/bazel:go_test_manifest_shard.sh"], args = [ + "--shard-offset=0", + "--shard-total=100", "$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", @@ -1454,6 +1461,44 @@ sh_test( tags = ["embedded"], ) +# Shards 51-100 of the manifest's 100-shard block; see bd_embedded_test. +# +# Tags: +# embedded: as bd_embedded_test. +sh_test( + name = "bd_embedded_part2_test", + size = "large", + # -test.timeout=19m and env: as bd_embedded_test. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_manifest_shard.sh"], + args = [ + "--shard-offset=50", + "--shard-total=100", + "$(rootpath //:.github/scripts/embedded-test-shard.sh)", + "BEADS_TEST_CMD_BINARY", + "$(rootpath :bd_test)", + "-test.timeout=19m", + ], + data = [ + ":bd", + ":bd_test", + ":go_srcs", + "//:.github/scripts/embedded-cmd-test-shards.txt", + "//:.github/scripts/embedded-test-shard.sh", + "//internal/storage/embeddeddolt/cmd:cmd_norace", + "@go_sdk//:bin/gofmt", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", + "BEADS_TEST_EMBEDDED_DOLT": "1", + "BEADS_TEST_EMBEDDED_SCHEMA_TOOL": "$(rlocationpath //internal/storage/embeddeddolt/cmd:cmd_norace)", + "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + "GORACE": "atexit_sleep_ms=0", + }, + shard_count = 50, + tags = ["embedded"], +) + # Runs 34 Bazel shards against the TestProxiedServer*/TestServerMode* # functions, assigned by the manifest's 34-shard block (duration-balanced; # see scripts/ci/proxied_test_durations.json), against this configuration's diff --git a/cmd/bd/autocommit_embedded_test.go b/cmd/bd/autocommit_embedded_test.go index 1da1e4cc1..146c45889 100644 --- a/cmd/bd/autocommit_embedded_test.go +++ b/cmd/bd/autocommit_embedded_test.go @@ -191,7 +191,10 @@ Created while Dolt auto-commit is in batch mode. }) } -func TestEmbeddedRoutedSiblingWritesCommitTargetHead(t *testing.T) { +// TestEmbeddedRoutedSiblingWritesCommitTargetHead* each run one routed sibling +// write (comment, note, reopen) from the source rig and assert it commits on +// the target rig's head; one top-level test per write for CI shard balance. +func TestEmbeddedRoutedSiblingWritesCommitTargetHeadComment(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") } @@ -223,6 +226,15 @@ func TestEmbeddedRoutedSiblingWritesCommitTargetHead(t *testing.T) { t.Fatal("routed comment was not persisted in target store") } }) +} + +func TestEmbeddedRoutedSiblingWritesCommitTargetHeadNote(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("note", func(t *testing.T) { sourceDir, targetDir, targetBeadsDir := setupRoutedEmbeddedRepo(t, bd, "sn", "tn") @@ -241,6 +253,15 @@ func TestEmbeddedRoutedSiblingWritesCommitTargetHead(t *testing.T) { t.Fatalf("target notes = %q, want routed note", got.Notes) } }) +} + +func TestEmbeddedRoutedSiblingWritesCommitTargetHeadReopen(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("reopen", func(t *testing.T) { sourceDir, targetDir, targetBeadsDir := setupRoutedEmbeddedRepo(t, bd, "sr", "tr") @@ -262,21 +283,20 @@ func TestEmbeddedRoutedSiblingWritesCommitTargetHead(t *testing.T) { }) } -// TestEmbeddedRoutedMutatingSiblingWritesCommitTargetHead pins the rest of the -// prefix-routed mutating command surface to the same write-through contract as +// TestEmbeddedRoutedMutatingSibling* pin the rest of the prefix-routed +// mutating command surface to the same write-through contract as // comment/note/reopen above. These commands (the assign/tag shorthands plus // dep/delete/close) resolve a prefix-routed target through the write-intent // router and must commit on the target head, not fail "store is read-only" // (#4141). Each subtest creates the bead in the target rig, runs the command // from the source rig so resolution must route, and asserts both that the // target HEAD advanced and that the mutation persisted in the target store. -// TestEmbeddedRoutedMutatingSiblingAssignTagDep was split from -// TestEmbeddedRoutedMutatingSiblingWritesCommitTargetHead (originally -// ~245.22s, measured under --config=embedded) into 2 top-level tests over -// disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedRoutedMutatingSiblingAssignTagDep(t *testing.T) { +// Each top-level test runs one subtest of the original +// TestEmbeddedRoutedMutatingSiblingWritesCommitTargetHead (later +// ...AssignTagDep and ...DeleteClose), for CI shard balance (see +// scripts/ci/embedded_cmd_test_durations.json): every original subtest is +// preserved exactly once. +func TestEmbeddedRoutedMutatingSiblingAssignTagDepAssign(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") } @@ -301,6 +321,15 @@ func TestEmbeddedRoutedMutatingSiblingAssignTagDep(t *testing.T) { t.Fatalf("target assignee = %q, want alice", got.Assignee) } }) +} + +func TestEmbeddedRoutedMutatingSiblingAssignTagDepTag(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("tag", func(t *testing.T) { sourceDir, targetDir, targetBeadsDir := setupRoutedEmbeddedRepo(t, bd, "stg", "ttg") @@ -319,6 +348,15 @@ func TestEmbeddedRoutedMutatingSiblingAssignTagDep(t *testing.T) { t.Fatalf("target labels = %v, want routed-label", labels[issue.ID]) } }) +} + +func TestEmbeddedRoutedMutatingSiblingAssignTagDepDepAdd(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("dep_add", func(t *testing.T) { sourceDir, targetDir, targetBeadsDir := setupRoutedEmbeddedRepo(t, bd, "sdp", "tdp") @@ -347,13 +385,7 @@ func TestEmbeddedRoutedMutatingSiblingAssignTagDep(t *testing.T) { }) } -// TestEmbeddedRoutedMutatingSiblingDeleteClose was split from -// TestEmbeddedRoutedMutatingSiblingWritesCommitTargetHead (originally -// ~245.22s, measured under --config=embedded) into 2 top-level tests over -// disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedRoutedMutatingSiblingDeleteClose(t *testing.T) { +func TestEmbeddedRoutedMutatingSiblingDeleteCloseDelete(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") } @@ -374,6 +406,15 @@ func TestEmbeddedRoutedMutatingSiblingDeleteClose(t *testing.T) { t.Fatalf("routed delete did not remove %s from target store", issue.ID) } }) +} + +func TestEmbeddedRoutedMutatingSiblingDeleteCloseClose(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("close", func(t *testing.T) { sourceDir, targetDir, targetBeadsDir := setupRoutedEmbeddedRepo(t, bd, "scl", "tcl") diff --git a/cmd/bd/bootstrap_embedded_test.go b/cmd/bd/bootstrap_embedded_test.go index 41a028802..e4f32a01d 100644 --- a/cmd/bd/bootstrap_embedded_test.go +++ b/cmd/bd/bootstrap_embedded_test.go @@ -88,7 +88,7 @@ func TestBootstrapNoWorkspace(t *testing.T) { }) } -func TestEmbeddedBootstrap(t *testing.T) { +func TestEmbeddedBootstrapExistingDryRunInit(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -160,6 +160,15 @@ func TestEmbeddedBootstrap(t *testing.T) { t.Errorf("expected 'Created fresh database':\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) } }) +} + +func TestEmbeddedBootstrapJSONLAndGitOrigin(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) // ===== JSONL Import ===== diff --git a/cmd/bd/children_embedded_test.go b/cmd/bd/children_embedded_test.go index 8e1b26851..d2dd4c4fc 100644 --- a/cmd/bd/children_embedded_test.go +++ b/cmd/bd/children_embedded_test.go @@ -26,7 +26,7 @@ func bdChildren(t *testing.T, bd, dir string, args ...string) string { return stdout.String() } -func TestEmbeddedChildren(t *testing.T) { +func TestEmbeddedChildrenBasicAndEmpty(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -51,6 +51,33 @@ func TestEmbeddedChildren(t *testing.T) { } }) + t.Run("children_empty", func(t *testing.T) { + parent := bdCreate(t, bd, dir, "No children parent", "--type", "task") + out := bdChildren(t, bd, dir, parent.ID) + // Should not error, may show empty message + _ = out + }) + + t.Run("children_nonexistent_parent", func(t *testing.T) { + cmd := exec.Command(bd, "children", "ch-nonexistent999") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("expected children of nonexistent to fail, got: %s", out) + } + }) +} + +func TestEmbeddedChildrenJSONAndAllStatuses(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "ch") + t.Run("children_json", func(t *testing.T) { parent := bdCreate(t, bd, dir, "JSON parent", "--type", "epic") child := bdCreate(t, bd, dir, "JSON child", "--type", "task") @@ -76,13 +103,6 @@ func TestEmbeddedChildren(t *testing.T) { } }) - t.Run("children_empty", func(t *testing.T) { - parent := bdCreate(t, bd, dir, "No children parent", "--type", "task") - out := bdChildren(t, bd, dir, parent.ID) - // Should not error, may show empty message - _ = out - }) - t.Run("children_includes_all_statuses", func(t *testing.T) { parent := bdCreate(t, bd, dir, "All status parent", "--type", "epic") openChild := bdCreate(t, bd, dir, "Open child", "--type", "task") @@ -99,16 +119,6 @@ func TestEmbeddedChildren(t *testing.T) { t.Errorf("expected closed child in output (--all implied): %s", out) } }) - - t.Run("children_nonexistent_parent", func(t *testing.T) { - cmd := exec.Command(bd, "children", "ch-nonexistent999") - cmd.Dir = dir - cmd.Env = bdEnv(dir) - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("expected children of nonexistent to fail, got: %s", out) - } - }) } // TestEmbeddedChildrenConcurrent exercises children listing concurrently. diff --git a/cmd/bd/close_embedded_test.go b/cmd/bd/close_embedded_test.go index 88ed651d1..2c87dbac0 100644 --- a/cmd/bd/close_embedded_test.go +++ b/cmd/bd/close_embedded_test.go @@ -58,11 +58,15 @@ func bdDepAdd(t *testing.T, bd, dir string, args ...string) { // ===== Close tests ===== -// TestEmbeddedCloseBasic was split from TestEmbeddedClose (originally ~404s, -// measured under --config=embedded) into 3 top-level tests over disjoint -// subtest groups, for CI shard balance (see scripts/ci/embedded_cmd_test_durations.json and -// engdocs/TESTING.md). Every original subtest is preserved exactly once. -func TestEmbeddedCloseBasic(t *testing.T) { +// The TestEmbeddedCloseBasic*, TestEmbeddedCloseGuardsAndEpics and +// TestEmbeddedCloseAlreadyClosed* tests were split from TestEmbeddedClose +// (originally ~404s, measured under --config=embedded) into top-level tests +// over disjoint subtest groups, for CI shard balance (see +// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every +// original subtest is preserved exactly once, in its original order within +// its group. Subtests that share a store stay in one test; the others each +// init their own isolated store. +func TestEmbeddedCloseBasicSharedStore(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -137,6 +141,15 @@ func TestEmbeddedCloseBasic(t *testing.T) { t.Errorf("blocked issue status = %s, want open", got.Status) } }) +} + +func TestEmbeddedCloseBasicIsolatedStores(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) // The direct-route twin of TestProxiedClose/ // close_partial_failure_json_names_the_failed_ids. Two things were asserted @@ -391,12 +404,8 @@ func TestEmbeddedCloseGuardsAndEpics(t *testing.T) { } -// TestEmbeddedCloseAlreadyClosed was split from TestEmbeddedClose (originally -// ~404s, measured under --config=embedded) into 3 top-level tests over -// disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedCloseAlreadyClosed(t *testing.T) { +// See TestEmbeddedCloseBasicSharedStore for the TestEmbeddedClose split. +func TestEmbeddedCloseAlreadyClosedMultipleIDs(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -457,6 +466,15 @@ func TestEmbeddedCloseAlreadyClosed(t *testing.T) { } } }) +} + +func TestEmbeddedCloseAlreadyClosedContinue(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("close_already_closed_continue_advances", func(t *testing.T) { // Isolated store so molecule progress and the Dolt commit count are @@ -527,6 +545,15 @@ func TestEmbeddedCloseAlreadyClosed(t *testing.T) { beforeCommits, afterCommits) } }) +} + +func TestEmbeddedCloseAlreadyClosedClaimNextAndMolecule(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("close_already_closed_claim_next", func(t *testing.T) { // Isolated store so the ready set is deterministic — the shared store carries diff --git a/cmd/bd/contributor_embedded_test.go b/cmd/bd/contributor_embedded_test.go index 71d31c0a5..8d58b24e1 100644 --- a/cmd/bd/contributor_embedded_test.go +++ b/cmd/bd/contributor_embedded_test.go @@ -13,11 +13,11 @@ import ( "github.com/steveyegge/beads/internal/storage/embeddeddolt" ) -// TestEmbeddedContributorCreate exercises the full contributor mode flow in +// The TestEmbeddedContributorCreate* tests exercise the full contributor mode flow in // embedded Dolt: init → contributor wizard → create. This is the exact // scenario that triggered GH#2988 ("no database selected" when the planning // repo's .beads directory has no metadata.json). -func TestEmbeddedContributorCreate(t *testing.T) { +func TestEmbeddedContributorCreateRoutesToPlanning(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt contributor tests") } @@ -46,6 +46,15 @@ func TestEmbeddedContributorCreate(t *testing.T) { projectBeadsDir := filepath.Join(dir, ".beads") assertIssueNotInStore(t, projectBeadsDir, "cr", issue.ID) }) +} + +func TestEmbeddedContributorCreateShowReadsRouted(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt contributor tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("show_reads_routed_issue", func(t *testing.T) { dir, _ := initContributor(t, bd, "sh") @@ -56,6 +65,15 @@ func TestEmbeddedContributorCreate(t *testing.T) { t.Errorf("title: got %q, want %q", shown.Title, "Show me") } }) +} + +func TestEmbeddedContributorCreateMultiple(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt contributor tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("multiple_creates_succeed", func(t *testing.T) { dir, planningDir := initContributor(t, bd, "mc") diff --git a/cmd/bd/delete_embedded_test.go b/cmd/bd/delete_embedded_test.go index 1ca59fead..5f224d21a 100644 --- a/cmd/bd/delete_embedded_test.go +++ b/cmd/bd/delete_embedded_test.go @@ -51,14 +51,22 @@ func bdShowFail(t *testing.T, bd, dir, id string) string { return string(out) } -func TestEmbeddedDelete(t *testing.T) { +// setupEmbeddedDelete is the TestEmbeddedDelete* tests' workspace (prefix td). +func setupEmbeddedDelete(t *testing.T) (string, string) { + t.Helper() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "td") + return bd, dir +} + +func TestEmbeddedDeleteBasic(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() - bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "td") + bd, dir := setupEmbeddedDelete(t) t.Run("delete_single_issue", func(t *testing.T) { issue := bdCreate(t, bd, dir, "Delete me", "--type", "task") @@ -80,6 +88,19 @@ func TestEmbeddedDelete(t *testing.T) { } }) + t.Run("delete_nonexistent", func(t *testing.T) { + bdDeleteFail(t, bd, dir, "td-nonexistent999", "--force") + }) +} + +func TestEmbeddedDeletePreviewAndGuards(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedDelete(t) + t.Run("delete_without_force_shows_preview", func(t *testing.T) { target := bdCreate(t, bd, dir, "Lonely", "--type", "task") @@ -158,6 +179,15 @@ func TestEmbeddedDelete(t *testing.T) { t.Fatalf("forced dry-run removed dependent: got %v, want %q", got["id"], child.ID) } }) +} + +func TestEmbeddedDeleteForceAndBatch(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedDelete(t) t.Run("delete_force_orphans_dependents", func(t *testing.T) { parent := bdCreate(t, bd, dir, "Force parent", "--type", "task") @@ -183,10 +213,6 @@ func TestEmbeddedDelete(t *testing.T) { bdShowFail(t, bd, dir, issue2.ID) bdShowFail(t, bd, dir, issue3.ID) }) - - t.Run("delete_nonexistent", func(t *testing.T) { - bdDeleteFail(t, bd, dir, "td-nonexistent999", "--force") - }) } func TestEmbeddedDeleteJSONDependencyErrorContract(t *testing.T) { diff --git a/cmd/bd/dep_embedded_test.go b/cmd/bd/dep_embedded_test.go index 378afa7ec..187255af9 100644 --- a/cmd/bd/dep_embedded_test.go +++ b/cmd/bd/dep_embedded_test.go @@ -97,17 +97,18 @@ func bdDepWithInputFail(t *testing.T, bd, dir, input string, args ...string) str return string(out) } -// TestEmbeddedDepA and TestEmbeddedDepB were split from TestEmbeddedDep -// (originally ~317s, measured under --config=embedded) into 2 top-level -// tests over disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. The "list_*" subtests (which assert on -// dependencies added by "add_positional_args" / "add_type_parent_child") -// stay grouped with the "add_*" cluster in TestEmbeddedDepA rather than a -// strict half/half split, since they depend on mutations made by those -// specific earlier subtests that a fresh per-function setup does not -// reproduce. -func TestEmbeddedDepA(t *testing.T) { +// The TestEmbeddedDepA* and TestEmbeddedDepB* tests were split from +// TestEmbeddedDep (originally ~317s, measured under --config=embedded), first +// into TestEmbeddedDepA and TestEmbeddedDepB and then (~95s each on +// rbe-west) into these 6 top-level tests over disjoint subtest groups, for +// CI shard balance (see scripts/ci/embedded_cmd_test_durations.json and +// engdocs/TESTING.md). Every original subtest is preserved exactly once. The +// "list_*" subtests (which assert on dependencies added by +// "add_positional_args" / "add_type_parent_child") stay with those subtests +// and the shared pre-created issues in TestEmbeddedDepAAddAndList, and the +// tree_* subtests with the pre-wired graph in TestEmbeddedDepBTreeAndCycles; +// the subtests that create every issue they use run in their own workspace. +func TestEmbeddedDepAAddAndList(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -153,6 +154,54 @@ func TestEmbeddedDepA(t *testing.T) { } }) + // ===== dep list ===== + + t.Run("list_default_direction_down", func(t *testing.T) { + // issueC depends on issueD (added earlier) + out := bdDep(t, bd, dir, "list", issueC.ID) + if !strings.Contains(out, issueD.ID) { + t.Errorf("expected dependency in list output: %s", out) + } + }) + + t.Run("list_direction_up", func(t *testing.T) { + out := bdDep(t, bd, dir, "list", issueD.ID, "--direction", "up") + if !strings.Contains(out, issueC.ID) { + t.Errorf("expected dependent in list --direction up: %s", out) + } + }) + + t.Run("list_type_filter", func(t *testing.T) { + out := bdDep(t, bd, dir, "list", epic.ID, "--direction", "up", "--type", "parent-child") + if !strings.Contains(out, child1.ID) || !strings.Contains(out, child2.ID) { + t.Errorf("expected children in type-filtered list: %s", out) + } + }) + + t.Run("list_json_output", func(t *testing.T) { + fullArgs := []string{"dep", "list", issueC.ID, "--json"} + cmd := exec.Command(bd, fullArgs...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("dep list --json failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + if !strings.Contains(stdout.String(), issueD.ID) { + t.Errorf("expected dependency ID in JSON: %s", stdout.String()) + } + }) +} + +func TestEmbeddedDepAAddTypesAndFlags(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "dp") + t.Run("add_type_tracks", func(t *testing.T) { tracker := bdCreate(t, bd, dir, "Tracker", "--type", "task") tracked := bdCreate(t, bd, dir, "Tracked", "--type", "task") @@ -189,6 +238,16 @@ func TestEmbeddedDepA(t *testing.T) { t.Errorf("expected dependency in deps: %s", out) } }) +} + +func TestEmbeddedDepAAddRejectionsAndBulk(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "dp") t.Run("add_cycle_rejected", func(t *testing.T) { // A->B already exists, add B->A to create cycle — should be rejected @@ -261,47 +320,9 @@ func TestEmbeddedDepA(t *testing.T) { t.Fatalf("bulk validation failure should not add valid rows: %s", list) } }) - - // ===== dep list ===== - - t.Run("list_default_direction_down", func(t *testing.T) { - // issueC depends on issueD (added earlier) - out := bdDep(t, bd, dir, "list", issueC.ID) - if !strings.Contains(out, issueD.ID) { - t.Errorf("expected dependency in list output: %s", out) - } - }) - - t.Run("list_direction_up", func(t *testing.T) { - out := bdDep(t, bd, dir, "list", issueD.ID, "--direction", "up") - if !strings.Contains(out, issueC.ID) { - t.Errorf("expected dependent in list --direction up: %s", out) - } - }) - - t.Run("list_type_filter", func(t *testing.T) { - out := bdDep(t, bd, dir, "list", epic.ID, "--direction", "up", "--type", "parent-child") - if !strings.Contains(out, child1.ID) || !strings.Contains(out, child2.ID) { - t.Errorf("expected children in type-filtered list: %s", out) - } - }) - - t.Run("list_json_output", func(t *testing.T) { - fullArgs := []string{"dep", "list", issueC.ID, "--json"} - cmd := exec.Command(bd, fullArgs...) - cmd.Dir = dir - cmd.Env = bdEnv(dir) - stdout, stderr, err := runCommandBuffers(t, cmd) - if err != nil { - t.Fatalf("dep list --json failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) - } - if !strings.Contains(stdout.String(), issueD.ID) { - t.Errorf("expected dependency ID in JSON: %s", stdout.String()) - } - }) } -func TestEmbeddedDepB(t *testing.T) { +func TestEmbeddedDepBTreeAndCycles(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -327,54 +348,6 @@ func TestEmbeddedDepB(t *testing.T) { bdDep(t, bd, dir, "add", child2.ID, epic.ID, "--type", "parent-child") bdDep(t, bd, dir, "add", issueC.ID, issueD.ID) - // ===== dep remove ===== - - t.Run("remove_basic", func(t *testing.T) { - r1 := bdCreate(t, bd, dir, "Remove A", "--type", "task") - r2 := bdCreate(t, bd, dir, "Remove B", "--type", "task") - bdDep(t, bd, dir, "add", r1.ID, r2.ID) - out := bdDep(t, bd, dir, "remove", r1.ID, r2.ID) - if !strings.Contains(out, "Removed") { - t.Errorf("expected 'Removed' in output: %s", out) - } - }) - - t.Run("remove_rm_alias", func(t *testing.T) { - r1 := bdCreate(t, bd, dir, "Rm A", "--type", "task") - r2 := bdCreate(t, bd, dir, "Rm B", "--type", "task") - bdDep(t, bd, dir, "add", r1.ID, r2.ID) - out := bdDep(t, bd, dir, "rm", r1.ID, r2.ID) - if !strings.Contains(out, "Removed") { - t.Errorf("expected 'Removed' via rm alias: %s", out) - } - }) - - t.Run("remove_json_output", func(t *testing.T) { - r1 := bdCreate(t, bd, dir, "RmJSON A", "--type", "task") - r2 := bdCreate(t, bd, dir, "RmJSON B", "--type", "task") - bdDep(t, bd, dir, "add", r1.ID, r2.ID) - m := bdDepJSON(t, bd, dir, "remove", r1.ID, r2.ID) - if m["status"] != "removed" { - t.Errorf("expected status=removed, got %v", m["status"]) - } - if m["removed"] != true { - t.Errorf("expected removed=true, got %v", m["removed"]) - } - }) - - t.Run("remove_missing_edge_reports_noop", func(t *testing.T) { - r1 := bdCreate(t, bd, dir, "Missing edge A", "--type", "task") - r2 := bdCreate(t, bd, dir, "Missing edge B", "--type", "task") - out := bdDep(t, bd, dir, "remove", r1.ID, r2.ID) - if !strings.Contains(out, "No dependency found") || strings.Contains(out, "✓") { - t.Errorf("missing edge should report a no-op, got: %s", out) - } - m := bdDepJSON(t, bd, dir, "remove", r1.ID, r2.ID) - if m["status"] != "not_found" || m["removed"] != false { - t.Errorf("missing edge JSON = %v, want status=not_found removed=false", m) - } - }) - // ===== dep tree ===== t.Run("tree_basic", func(t *testing.T) { @@ -433,6 +406,85 @@ func TestEmbeddedDepB(t *testing.T) { } }) + // ===== dep cycles ===== + + t.Run("cycles_detect", func(t *testing.T) { + // We created a cycle earlier (cyA <-> cyB) + out := bdDep(t, bd, dir, "cycles") + // Should report at least one cycle + if !strings.Contains(out, "cycle") && !strings.Contains(out, "Cycle") && !strings.Contains(out, "No cycles") { + t.Errorf("expected cycle info in output: %s", out) + } + }) +} + +func TestEmbeddedDepBRemove(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "dp") + + // ===== dep remove ===== + + t.Run("remove_basic", func(t *testing.T) { + r1 := bdCreate(t, bd, dir, "Remove A", "--type", "task") + r2 := bdCreate(t, bd, dir, "Remove B", "--type", "task") + bdDep(t, bd, dir, "add", r1.ID, r2.ID) + out := bdDep(t, bd, dir, "remove", r1.ID, r2.ID) + if !strings.Contains(out, "Removed") { + t.Errorf("expected 'Removed' in output: %s", out) + } + }) + + t.Run("remove_rm_alias", func(t *testing.T) { + r1 := bdCreate(t, bd, dir, "Rm A", "--type", "task") + r2 := bdCreate(t, bd, dir, "Rm B", "--type", "task") + bdDep(t, bd, dir, "add", r1.ID, r2.ID) + out := bdDep(t, bd, dir, "rm", r1.ID, r2.ID) + if !strings.Contains(out, "Removed") { + t.Errorf("expected 'Removed' via rm alias: %s", out) + } + }) +} + +func TestEmbeddedDepBRemoveOutputAndRelatesTo(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "dp") + + t.Run("remove_json_output", func(t *testing.T) { + r1 := bdCreate(t, bd, dir, "RmJSON A", "--type", "task") + r2 := bdCreate(t, bd, dir, "RmJSON B", "--type", "task") + bdDep(t, bd, dir, "add", r1.ID, r2.ID) + m := bdDepJSON(t, bd, dir, "remove", r1.ID, r2.ID) + if m["status"] != "removed" { + t.Errorf("expected status=removed, got %v", m["status"]) + } + if m["removed"] != true { + t.Errorf("expected removed=true, got %v", m["removed"]) + } + }) + + t.Run("remove_missing_edge_reports_noop", func(t *testing.T) { + r1 := bdCreate(t, bd, dir, "Missing edge A", "--type", "task") + r2 := bdCreate(t, bd, dir, "Missing edge B", "--type", "task") + out := bdDep(t, bd, dir, "remove", r1.ID, r2.ID) + if !strings.Contains(out, "No dependency found") || strings.Contains(out, "✓") { + t.Errorf("missing edge should report a no-op, got: %s", out) + } + m := bdDepJSON(t, bd, dir, "remove", r1.ID, r2.ID) + if m["status"] != "not_found" || m["removed"] != false { + t.Errorf("missing edge JSON = %v, want status=not_found removed=false", m) + } + }) + t.Run("tree_ignores_bidirectional_relates_to", func(t *testing.T) { root := bdCreate(t, bd, dir, "Tree relates root", "--type", "task") blocker := bdCreate(t, bd, dir, "Tree real blocker", "--type", "task") @@ -461,17 +513,6 @@ func TestEmbeddedDepB(t *testing.T) { } }) - // ===== dep cycles ===== - - t.Run("cycles_detect", func(t *testing.T) { - // We created a cycle earlier (cyA <-> cyB) - out := bdDep(t, bd, dir, "cycles") - // Should report at least one cycle - if !strings.Contains(out, "cycle") && !strings.Contains(out, "Cycle") && !strings.Contains(out, "No cycles") { - t.Errorf("expected cycle info in output: %s", out) - } - }) - t.Run("cycles_no_cycles", func(t *testing.T) { // Fresh init with no cycles dir2, _, _ := bdInit(t, bd, "--prefix", "dp2") diff --git a/cmd/bd/duplicates_embedded_test.go b/cmd/bd/duplicates_embedded_test.go index d3275231a..aaf941f82 100644 --- a/cmd/bd/duplicates_embedded_test.go +++ b/cmd/bd/duplicates_embedded_test.go @@ -49,7 +49,7 @@ func bdDuplicatesJSON(t *testing.T, bd, dir string, args ...string) map[string]i return m } -func TestEmbeddedDuplicates(t *testing.T) { +func TestEmbeddedDuplicatesSharedFixture(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -76,6 +76,49 @@ func TestEmbeddedDuplicates(t *testing.T) { } }) + // ===== JSON output ===== + + t.Run("json_output_structure", func(t *testing.T) { + m := bdDuplicatesJSON(t, bd, dir) + if _, ok := m["duplicate_groups"]; !ok { + t.Error("expected 'duplicate_groups' key") + } + if _, ok := m["groups"]; !ok { + t.Error("expected 'groups' key") + } + }) + + // ===== Dry run ===== + + t.Run("dry_run", func(t *testing.T) { + m := bdDuplicatesJSON(t, bd, dir, "--dry-run") + if _, ok := m["merge_commands"]; !ok { + t.Error("expected 'merge_commands' key with --dry-run") + } + // Dry run should NOT have merge_results + if _, ok := m["merge_results"]; ok { + t.Error("dry run should not have merge_results") + } + }) + + // ===== Human-readable output ===== + + t.Run("human_readable", func(t *testing.T) { + out := bdDuplicates(t, bd, dir) + if !strings.Contains(out, "duplicate") && !strings.Contains(out, "Duplicate") && !strings.Contains(out, "No duplicates") { + t.Errorf("expected duplicate info in output: %s", out) + } + }) +} + +func TestEmbeddedDuplicatesOwnWorkspaces(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + // ===== No duplicates when all unique ===== t.Run("no_duplicates", func(t *testing.T) { @@ -89,18 +132,6 @@ func TestEmbeddedDuplicates(t *testing.T) { } }) - // ===== JSON output ===== - - t.Run("json_output_structure", func(t *testing.T) { - m := bdDuplicatesJSON(t, bd, dir) - if _, ok := m["duplicate_groups"]; !ok { - t.Error("expected 'duplicate_groups' key") - } - if _, ok := m["groups"]; !ok { - t.Error("expected 'groups' key") - } - }) - // ===== Auto-merge ===== t.Run("auto_merge", func(t *testing.T) { @@ -142,19 +173,6 @@ func TestEmbeddedDuplicates(t *testing.T) { } }) - // ===== Dry run ===== - - t.Run("dry_run", func(t *testing.T) { - m := bdDuplicatesJSON(t, bd, dir, "--dry-run") - if _, ok := m["merge_commands"]; !ok { - t.Error("expected 'merge_commands' key with --dry-run") - } - // Dry run should NOT have merge_results - if _, ok := m["merge_results"]; ok { - t.Error("dry run should not have merge_results") - } - }) - // ===== Excludes closed issues ===== t.Run("excludes_closed", func(t *testing.T) { @@ -170,15 +188,6 @@ func TestEmbeddedDuplicates(t *testing.T) { t.Log("mixed open/closed duplicates detected — expected 0 groups due to status mismatch") } }) - - // ===== Human-readable output ===== - - t.Run("human_readable", func(t *testing.T) { - out := bdDuplicates(t, bd, dir) - if !strings.Contains(out, "duplicate") && !strings.Contains(out, "Duplicate") && !strings.Contains(out, "No duplicates") { - t.Errorf("expected duplicate info in output: %s", out) - } - }) } // TestEmbeddedDuplicatesConcurrent exercises duplicates operations concurrently. diff --git a/cmd/bd/epic_embedded_test.go b/cmd/bd/epic_embedded_test.go index b2e064d7f..e1eb91403 100644 --- a/cmd/bd/epic_embedded_test.go +++ b/cmd/bd/epic_embedded_test.go @@ -119,12 +119,15 @@ func TestEmbeddedEpicStatus(t *testing.T) { }) } -// TestEmbeddedEpicCloseEligible was split from TestEmbeddedEpic (originally +// TestEmbeddedEpicCloseEligible* was split from TestEmbeddedEpic (originally // ~362s, measured under --config=embedded) into 3 top-level tests over // disjoint subtest groups, for CI shard balance (see // scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedEpicCloseEligible(t *testing.T) { +// subtest is preserved exactly once. It is split again in two: +// TestEmbeddedEpicCloseEligibleNotEligible keeps the shared not-eligible epic +// fixture and the two subtests that read it, and +// TestEmbeddedEpicCloseEligibleClosesEpics the two that build their own. +func TestEmbeddedEpicCloseEligibleNotEligible(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -156,6 +159,23 @@ func TestEmbeddedEpicCloseEligible(t *testing.T) { } }) + t.Run("close_eligible_none", func(t *testing.T) { + // epic1 has open children — should not be closeable + out := bdEpic(t, bd, dir, "close-eligible") + if strings.Contains(out, epic1.ID) { + t.Errorf("epic1 should not be closed: %s", out) + } + }) +} + +func TestEmbeddedEpicCloseEligibleClosesEpics(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + t.Run("close_eligible_closes_epics", func(t *testing.T) { dir3, _, _ := bdInit(t, bd, "--prefix", "ep3") e := bdCreate(t, bd, dir3, "Close me epic", "--type", "epic") @@ -202,14 +222,6 @@ func TestEmbeddedEpicCloseEligible(t *testing.T) { t.Fatalf("no JSON: %s", s) } }) - - t.Run("close_eligible_none", func(t *testing.T) { - // epic1 has open children — should not be closeable - out := bdEpic(t, bd, dir, "close-eligible") - if strings.Contains(out, epic1.ID) { - t.Errorf("epic1 should not be closed: %s", out) - } - }) } // TestEmbeddedEpicCloseEligibleReason was split from TestEmbeddedEpic diff --git a/cmd/bd/gate_embedded_test.go b/cmd/bd/gate_embedded_test.go index 04c93963c..d2c8030a8 100644 --- a/cmd/bd/gate_embedded_test.go +++ b/cmd/bd/gate_embedded_test.go @@ -97,11 +97,10 @@ func createBeadGate(t *testing.T, bd, dir, awaitID string) *types.Issue { return &gate } -func TestEmbeddedGate(t *testing.T) { - if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { - t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") - } - t.Parallel() +// setupEmbeddedGate is the TestEmbeddedGate* tests' workspace: prefix tg, +// with "gate" registered as a custom type. +func setupEmbeddedGate(t *testing.T) (string, string) { + t.Helper() bd := buildEmbeddedBD(t) dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tg") @@ -112,6 +111,16 @@ func TestEmbeddedGate(t *testing.T) { t.Fatalf("SetConfig types.custom: %v", err) } store.Close() + return bd, dir +} + +func TestEmbeddedGateList(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedGate(t) // ===== Gate List ===== @@ -173,6 +182,15 @@ func TestEmbeddedGate(t *testing.T) { t.Errorf("expected at most 1 result with --limit 1, got %d", len(results)) } }) +} + +func TestEmbeddedGateShowResolve(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedGate(t) // ===== Gate Show ===== @@ -233,6 +251,15 @@ func TestEmbeddedGate(t *testing.T) { task := bdCreate(t, bd, dir, "Not a gate resolve", "--type", "task") bdGateFail(t, bd, dir, "resolve", task.ID) }) +} + +func TestEmbeddedGateWaiterCheckLifecycle(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedGate(t) // ===== Gate Add-Waiter ===== @@ -375,12 +402,11 @@ func TestEmbeddedGate(t *testing.T) { }) } -// TestEmbeddedGateCreate exercises the "bd gate create" subcommand. -func TestEmbeddedGateCreate(t *testing.T) { - if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { - t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") - } - t.Parallel() +// setupEmbeddedGateCreate is the shared workspace of the +// TestEmbeddedGateCreate* tests ("bd gate create"): prefix gc, with "gate" +// registered as a custom type so bd gate create works. +func setupEmbeddedGateCreate(t *testing.T) (string, string) { + t.Helper() bd := buildEmbeddedBD(t) dir, beadsDir, _ := bdInit(t, bd, "--prefix", "gc") @@ -391,6 +417,16 @@ func TestEmbeddedGateCreate(t *testing.T) { t.Fatalf("SetConfig types.custom: %v", err) } store.Close() + return bd, dir +} + +func TestEmbeddedGateCreateFlags(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedGateCreate(t) t.Run("create_default_human_gate", func(t *testing.T) { task := bdCreate(t, bd, dir, "Task for human gate", "--type", "task") @@ -492,6 +528,15 @@ func TestEmbeddedGateCreate(t *testing.T) { t.Errorf("expected title 'Gate: gh:pr 42', got %s", gate.Title) } }) +} + +func TestEmbeddedGateCreateRepoErrorsList(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedGateCreate(t) t.Run("create_gate_with_repo", func(t *testing.T) { task := bdCreate(t, bd, dir, "Task for cross-repo PR gate", "--type", "task") @@ -536,6 +581,48 @@ func TestEmbeddedGateCreate(t *testing.T) { } }) + t.Run("create_gate_missing_blocks_flag", func(t *testing.T) { + out := bdGateFail(t, bd, dir, "create") + if !strings.Contains(out, "blocks") { + t.Errorf("expected error about missing --blocks flag: %s", out) + } + }) + + t.Run("create_gate_nonexistent_target", func(t *testing.T) { + out := bdGateFail(t, bd, dir, "create", "--blocks", "gc-nonexistent999") + if !strings.Contains(out, "not found") { + t.Errorf("expected 'not found' error: %s", out) + } + }) + + t.Run("create_gate_appears_in_gate_list", func(t *testing.T) { + task := bdCreate(t, bd, dir, "Task for list check", "--type", "task") + bdGate(t, bd, dir, "create", "--blocks", task.ID) + + results := bdGateListJSON(t, bd, dir) + found := false + for _, r := range results { + if awaitType, ok := r["await_type"]; ok && awaitType == "human" { + if desc, ok := r["description"].(string); ok && strings.Contains(desc, task.ID) { + found = true + break + } + } + } + if !found { + t.Errorf("expected gate blocking %s in gate list", task.ID) + } + }) +} + +func TestEmbeddedGateCreateReady(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + t.Run("create_gate_blocks_ready", func(t *testing.T) { // Use a fresh db so ready output isn't polluted by other subtests freshDir, freshBeads, _ := bdInit(t, bd, "--prefix", "gr") @@ -620,39 +707,15 @@ func TestEmbeddedGateCreate(t *testing.T) { t.Errorf("task should reappear in ready after gate resolved: %s", stdout.String()) } }) +} - t.Run("create_gate_missing_blocks_flag", func(t *testing.T) { - out := bdGateFail(t, bd, dir, "create") - if !strings.Contains(out, "blocks") { - t.Errorf("expected error about missing --blocks flag: %s", out) - } - }) +func TestEmbeddedGateCreateBeadCheck(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() - t.Run("create_gate_nonexistent_target", func(t *testing.T) { - out := bdGateFail(t, bd, dir, "create", "--blocks", "gc-nonexistent999") - if !strings.Contains(out, "not found") { - t.Errorf("expected 'not found' error: %s", out) - } - }) - - t.Run("create_gate_appears_in_gate_list", func(t *testing.T) { - task := bdCreate(t, bd, dir, "Task for list check", "--type", "task") - bdGate(t, bd, dir, "create", "--blocks", task.ID) - - results := bdGateListJSON(t, bd, dir) - found := false - for _, r := range results { - if awaitType, ok := r["await_type"]; ok && awaitType == "human" { - if desc, ok := r["description"].(string); ok && strings.Contains(desc, task.ID) { - found = true - break - } - } - } - if !found { - t.Errorf("expected gate blocking %s in gate list", task.ID) - } - }) + bd := buildEmbeddedBD(t) t.Run("check_bead_gate_resolves_only_a_missing_target_it_saw", func(t *testing.T) { // Use a fresh db so the check sees only these bead gates @@ -690,6 +753,15 @@ func TestEmbeddedGateCreate(t *testing.T) { t.Errorf("gate on a never-seen await_id: status=%s, want open", got.Status) } }) +} + +func TestEmbeddedGateCreateBeadRename(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("rename_keeps_a_bead_gate_on_its_target", func(t *testing.T) { // Renaming the awaited bead points its gate at the new ID and moves diff --git a/cmd/bd/gc_embedded_test.go b/cmd/bd/gc_embedded_test.go index 26f72588f..7d71e7751 100644 --- a/cmd/bd/gc_embedded_test.go +++ b/cmd/bd/gc_embedded_test.go @@ -39,7 +39,7 @@ func bdGCFail(t *testing.T, bd, dir string, args ...string) string { return string(out) } -func TestEmbeddedGC(t *testing.T) { +func TestEmbeddedGCDryRunAndSkips(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -98,6 +98,22 @@ func TestEmbeddedGC(t *testing.T) { } }) + t.Run("gc_full_dry_run", func(t *testing.T) { + out := bdGC(t, bd, dir, "--full", "--dry-run") + if !strings.Contains(out, "full DOLT_GC()") { + t.Errorf("dry run does not say it would run a full GC: %s", out) + } + }) +} + +func TestEmbeddedGCForceWithDecay(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + // ===== Force with Decay ===== t.Run("gc_force_with_decay", func(t *testing.T) { @@ -125,6 +141,15 @@ func TestEmbeddedGC(t *testing.T) { } } }) +} + +func TestEmbeddedGCFullAndNoForcePrompts(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) // ===== Full GC ===== @@ -147,13 +172,6 @@ func TestEmbeddedGC(t *testing.T) { } }) - t.Run("gc_full_dry_run", func(t *testing.T) { - out := bdGC(t, bd, dir, "--full", "--dry-run") - if !strings.Contains(out, "full DOLT_GC()") { - t.Errorf("dry run does not say it would run a full GC: %s", out) - } - }) - // ===== No --force prompts ===== t.Run("gc_no_force_prompts", func(t *testing.T) { diff --git a/cmd/bd/human_embedded_test.go b/cmd/bd/human_embedded_test.go index 51e3b7bc6..204a3cc4a 100644 --- a/cmd/bd/human_embedded_test.go +++ b/cmd/bd/human_embedded_test.go @@ -48,7 +48,7 @@ func humanShowClosed(t *testing.T, bd, dir, id string) *types.Issue { return issue } -func TestEmbeddedHuman(t *testing.T) { +func TestEmbeddedHumanListStatsRespondDismiss(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -127,6 +127,16 @@ func TestEmbeddedHuman(t *testing.T) { t.Errorf("expected dismiss reason %q, got %q", "Dismissed: Not needed", issue2.CloseReason) } }) +} + +func TestEmbeddedHumanPositionalFileAndGuards(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "th") // ===== Respond via positional text and --file ===== diff --git a/cmd/bd/init_embedded_test.go b/cmd/bd/init_embedded_test.go index 891985cd4..d5c4449b2 100644 --- a/cmd/bd/init_embedded_test.go +++ b/cmd/bd/init_embedded_test.go @@ -11,6 +11,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strings" "sync" "testing" @@ -338,12 +339,12 @@ func requireNoFile(t *testing.T, path string) { } } -// TestEmbeddedInitA, TestEmbeddedInitB, and TestEmbeddedInitC were split from -// TestEmbeddedInit (originally ~356s, measured under --config=embedded) into -// 3 top-level tests over disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedInitA(t *testing.T) { +// The TestEmbeddedInitA*, TestEmbeddedInitB* and TestEmbeddedInitC tests were +// split from TestEmbeddedInit (originally ~356s, measured under +// --config=embedded) into top-level tests over disjoint subtest groups, for CI +// shard balance (see scripts/ci/embedded_cmd_test_durations.json and +// engdocs/TESTING.md). Every original subtest is preserved exactly once. +func TestEmbeddedInitABasic(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") } @@ -478,6 +479,15 @@ func TestEmbeddedInitA(t *testing.T) { t.Errorf("issue_prefix: got %q, want %q", val, "alpha") } }) +} + +func TestEmbeddedInitAForkAutoContributor(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("fork_auto_contributor", func(t *testing.T) { dir := t.TempDir() @@ -541,6 +551,15 @@ func TestEmbeddedInitA(t *testing.T) { t.Errorf("beads.role: got %q, want %q", role, "contributor") } }) +} + +func TestEmbeddedInitAGitOriginRemote(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("git_origin_registered_as_dolt_remote", func(t *testing.T) { bareDir := filepath.Join(t.TempDir(), "plain.git") @@ -583,6 +602,15 @@ func TestEmbeddedInitA(t *testing.T) { t.Fatalf("bd dolt push did not publish refs/dolt/data:\n%s", lsOut) } }) +} + +func TestEmbeddedInitAPushConsentAndRemoteBootstrap(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) // The #5068 refusal and consent paths, end to end. @@ -768,7 +796,7 @@ func TestEmbeddedInitA(t *testing.T) { }) } -func TestEmbeddedInitB(t *testing.T) { +func TestEmbeddedInitBRemoteSchemaAndCloneFailure(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") } @@ -926,6 +954,15 @@ func TestEmbeddedInitB(t *testing.T) { t.Fatalf(".beads/config.yaml should not exist after a failed clone; init must not silently fall through to fresh init") } }) +} + +func TestEmbeddedInitBStealthAndForceReinit(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) // Regression: bd init --stealth must not touch any git-visible files. Previously it // created/modified the tracked project-root .gitignore via doctor.EnsureProjectGitignore, which @@ -1090,6 +1127,15 @@ func TestEmbeddedInitB(t *testing.T) { t.Errorf("issue_prefix after --force: got %q, want %q", val, "fi") } }) +} + +func TestEmbeddedInitBAutoCommitAndJSONLRemote(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("auto_commit_bypasses_hooks", func(t *testing.T) { dir := t.TempDir() @@ -1527,7 +1573,17 @@ func TestInitGateBusyClassifiedAsLockContention(t *testing.T) { } } -func TestEmbeddedInitRoleRouting(t *testing.T) { +// The role routing cases are two top-level tests, not one: each case is a +// full embedded init under -race and they cannot run in parallel (t.Setenv). +func TestEmbeddedInitRoleRoutingExplicitDefault(t *testing.T) { + testEmbeddedInitRoleRouting(t, "explicit", "default") +} + +func TestEmbeddedInitRoleRoutingRetainedFork(t *testing.T) { + testEmbeddedInitRoleRouting(t, "retained", "fork") +} + +func testEmbeddedInitRoleRouting(t *testing.T, names ...string) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") } @@ -1539,6 +1595,9 @@ func TestEmbeddedInitRoleRouting(t *testing.T) { {"retained", "contributor", "", "contributor"}, {"fork", "", "", "contributor"}, } { + if !slices.Contains(names, tc.name) { + continue + } t.Run(tc.name, func(t *testing.T) { target, decoy, home := newInitRoleFixture(t) if tc.initial != "" { @@ -1606,15 +1665,23 @@ func TestEmbeddedInitRoleRouting(t *testing.T) { } } -// The selected-exclude routing cases are two top-level tests, not one: each +// The selected-exclude routing cases are four top-level tests, not one: each // case is a full embedded init under -race and they cannot run in parallel // (t.Setenv), so one function held a whole Bazel shard for ~2 minutes. -func TestEmbeddedInitSelectedExcludeRoutingStealth(t *testing.T) { - testEmbeddedInitSelectedExcludeRouting(t, "stealth_decoy", "stealth_invalid", "quiet_stealth") +func TestEmbeddedInitSelectedExcludeRoutingStealthInvalid(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "stealth_invalid") } -func TestEmbeddedInitSelectedExcludeRoutingFork(t *testing.T) { - testEmbeddedInitSelectedExcludeRouting(t, "fork_decoy", "fork_invalid", "quiet_fork", "fork_auto_invalid") +func TestEmbeddedInitSelectedExcludeRoutingStealthDecoyQuiet(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "stealth_decoy", "quiet_stealth") +} + +func TestEmbeddedInitSelectedExcludeRoutingForkDecoyAuto(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "fork_decoy", "fork_auto_invalid") +} + +func TestEmbeddedInitSelectedExcludeRoutingForkInvalidQuiet(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "fork_invalid", "quiet_fork") } func testEmbeddedInitSelectedExcludeRouting(t *testing.T, names ...string) { @@ -1744,12 +1811,27 @@ func TestEmbeddedInitArtifactRouting(t *testing.T) { } } -func TestEmbeddedInitGitBootstrapRouting(t *testing.T) { +// The git bootstrap routing cases are three top-level tests, not one: each +// case is a full embedded init under -race and they cannot run in parallel +// (t.Setenv). +func TestEmbeddedInitGitBootstrapRoutingFreshQuiet(t *testing.T) { + testEmbeddedInitGitBootstrapRouting(t, "fresh", "quiet") +} + +func TestEmbeddedInitGitBootstrapRoutingDecoyInvalid(t *testing.T) { + testEmbeddedInitGitBootstrapRouting(t, "decoy", "invalid") +} + +func TestEmbeddedInitGitBootstrapRoutingExistingExplicit(t *testing.T) { + testEmbeddedInitGitBootstrapRouting(t, "existing_invalid", "explicit_storage") +} + +func testEmbeddedInitGitBootstrapRouting(t *testing.T, names ...string) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") } bd := buildEmbeddedBD(t) - for _, name := range []string{"fresh", "decoy", "invalid", "existing_invalid", "explicit_storage", "quiet"} { + for _, name := range names { t.Run(name, func(t *testing.T) { existing, decoy, home := newInitRoleFixture(t) target := t.TempDir() diff --git a/cmd/bd/label_embedded_test.go b/cmd/bd/label_embedded_test.go index e4f1e88a4..e34e6fcfa 100644 --- a/cmd/bd/label_embedded_test.go +++ b/cmd/bd/label_embedded_test.go @@ -125,12 +125,15 @@ func markIssueTemplate(t *testing.T, beadsDir, database, issueID string) { } } -// TestEmbeddedLabelAddRemove was split from TestEmbeddedLabel (originally -// ~360s, measured under --config=embedded) into 3 top-level tests over -// disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedLabelAddRemove(t *testing.T) { +// The TestEmbeddedLabelAddRemove* and TestEmbeddedLabelEditReports* tests +// were split from TestEmbeddedLabel (originally ~360s, measured under +// --config=embedded) into top-level tests over disjoint subtest groups, each +// on its own workspace, for CI shard balance (see +// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every +// original subtest is preserved exactly once, in its original order within +// its group; the label list-all subtests still follow subtests that add +// labels. +func TestEmbeddedLabelAddRemoveAdd(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -189,6 +192,16 @@ func TestEmbeddedLabelAddRemove(t *testing.T) { t.Errorf("expected valid JSON: %s", s) } }) +} + +func TestEmbeddedLabelAddRemoveMultiAndDuplicate(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tl") t.Run("label_add_comma_separated_multi", func(t *testing.T) { issue := bdCreate(t, bd, dir, "Multi label add", "--type", "task") @@ -244,6 +257,16 @@ func TestEmbeddedLabelAddRemove(t *testing.T) { t.Errorf("expected exactly 1 'dup' label, got %d in %v", count, labels) } }) +} + +func TestEmbeddedLabelAddRemoveRemove(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tl") // ===== Label Remove ===== @@ -375,19 +398,15 @@ func TestEmbeddedLabelAddRemove(t *testing.T) { }) } -// TestEmbeddedLabelEditReports was split from TestEmbeddedLabel (originally -// ~360s, measured under --config=embedded) into 3 top-level tests over -// disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedLabelEditReports(t *testing.T) { +// See TestEmbeddedLabelAddRemoveAdd for the TestEmbeddedLabel split. +func TestEmbeddedLabelEditReportsPrefix(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() bd := buildEmbeddedBD(t) - dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tl") + dir, _, _ := bdInit(t, bd, "--prefix", "tl") // Two issues carrying DIFFERENT matching sets: the report is per issue and // each issue's set is resolved from its own labels, so a refactor that @@ -476,6 +495,16 @@ func TestEmbeddedLabelEditReports(t *testing.T) { t.Errorf("expected valid JSON: %s", s) } }) +} + +func TestEmbeddedLabelEditReportsNoop(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tl") // ===== No-op edits (GH#5988) ===== // @@ -584,6 +613,16 @@ func TestEmbeddedLabelEditReports(t *testing.T) { t.Errorf("add claimed an edit on the issue that already had the label: %s", out) } }) +} + +func TestEmbeddedLabelEditReportsListAndTemplate(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tl") // ===== Label List ===== diff --git a/cmd/bd/list_embedded_test.go b/cmd/bd/list_embedded_test.go index d67b026e1..231c253ae 100644 --- a/cmd/bd/list_embedded_test.go +++ b/cmd/bd/list_embedded_test.go @@ -177,22 +177,33 @@ type testSeedData struct { readyTask string // P0, labels: backend, no blockers } -// TestEmbeddedListFiltering and TestEmbeddedListSearchAndOutput were split -// from TestEmbeddedList (originally ~298s, measured under --config=embedded) -// into 2 top-level tests over disjoint subtest groups, for CI shard balance -// (see scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedListFiltering(t *testing.T) { +// seededListWorkspace is the TestEmbeddedList* tests' shared setup: an +// initialized workspace seeded by seedTestData. +func seededListWorkspace(t *testing.T) (bd, dir string, seed testSeedData) { + t.Helper() + bd = buildEmbeddedBD(t) + dir, _, _ = bdInit(t, bd, "--prefix", "tl") + + // Seed test data + seed = seedTestData(t, bd, dir) + return bd, dir, seed +} + +// The TestEmbeddedListFiltering* and TestEmbeddedListSearchAndOutput* tests +// were split from TestEmbeddedList (originally ~298s, measured under +// --config=embedded) into top-level tests over disjoint subtest groups, each +// on its own seeded workspace, for CI shard balance (see +// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every +// original subtest is preserved exactly once, in its original order within +// its group; each group ran on the unmodified seed in the original too (the +// subtests before each split point create no issues). +func TestEmbeddedListFilteringBasicAndLabels(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() - bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "tl") - - // Seed test data - seed := seedTestData(t, bd, dir) + bd, dir, seed := seededListWorkspace(t) // --- A. Basic filtering --- @@ -454,6 +465,15 @@ func TestEmbeddedListFiltering(t *testing.T) { t.Fatalf("--skip-labels JSON leaked labels: %v", out.Issues[0].Labels) } }) +} + +func TestEmbeddedListFilteringStatusAndHierarchy(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir, seed := seededListWorkspace(t) // --- C. Status/special filtering --- // Note: --ready, --pinned, --status closed/deferred/in_progress tests are @@ -612,17 +632,13 @@ func TestEmbeddedListFiltering(t *testing.T) { }) } -func TestEmbeddedListSearchAndOutput(t *testing.T) { +func TestEmbeddedListSearchAndOutputSearchAndSort(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() - bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "tl") - - // Seed test data - seed := seedTestData(t, bd, dir) + bd, dir, seed := seededListWorkspace(t) // --- E. Content search --- @@ -725,6 +741,15 @@ func TestEmbeddedListSearchAndOutput(t *testing.T) { } } }) +} + +func TestEmbeddedListSearchAndOutputFormats(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir, seed := seededListWorkspace(t) // --- I. Output formats --- diff --git a/cmd/bd/ready_embedded_test.go b/cmd/bd/ready_embedded_test.go index ca6b29839..2e35f8980 100644 --- a/cmd/bd/ready_embedded_test.go +++ b/cmd/bd/ready_embedded_test.go @@ -16,7 +16,7 @@ import ( "github.com/steveyegge/beads/internal/types" ) -func TestEmbeddedReady(t *testing.T) { +func TestEmbeddedReadyDefaultJSONAndFlags(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -193,125 +193,6 @@ func TestEmbeddedReady(t *testing.T) { } }) - // ===== With Blockers ===== - - t.Run("ready_excludes_blocked", func(t *testing.T) { - blocker := bdCreate(t, bd, dir, "Blocker issue", "--type", "task") - blocked := bdCreate(t, bd, dir, "Blocked by blocker", "--type", "task") - - // Add blocking dependency: blocked depends on blocker - cmd := exec.Command(bd, "dep", "add", blocked.ID, blocker.ID) - cmd.Dir = dir - cmd.Env = bdEnv(dir) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("dep add failed: %v\n%s", err, out) - } - - cmd = exec.Command(bd, "ready") - cmd.Dir = dir - cmd.Env = bdEnv(dir) - stdout, stderr, err := runCommandBuffers(t, cmd) - if err != nil { - t.Fatalf("bd ready failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) - } - // The blocked issue should not appear in ready output - if strings.Contains(stdout.String(), "Blocked by blocker") { - t.Errorf("blocked issue should not appear in ready output: %s", stdout.String()) - } - }) - - // ===== Exclude Label ===== - - t.Run("ready_exclude_label", func(t *testing.T) { - bdCreate(t, bd, dir, "Triage pending item", "--type", "task", "--label", "triage:pending") - bdCreate(t, bd, dir, "Normal ready item", "--type", "task") - - cmd := exec.Command(bd, "ready", "--exclude-label", "triage:pending") - cmd.Dir = dir - cmd.Env = bdEnv(dir) - stdout, stderr, err := runCommandBuffers(t, cmd) - if err != nil { - t.Fatalf("bd ready --exclude-label failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) - } - if strings.Contains(stdout.String(), "Triage pending item") { - t.Errorf("triage:pending issue should not appear with --exclude-label: %s", stdout.String()) - } - if !strings.Contains(stdout.String(), "Normal ready item") { - t.Errorf("normal issue should still appear with --exclude-label: %s", stdout.String()) - } - }) - - // ===== Label Any ===== - - readyIDs := func(t *testing.T, args ...string) ([]string, string) { - t.Helper() - cmd := exec.Command(bd, args...) - cmd.Dir = dir - cmd.Env = bdEnv(dir) - stdout, stderr, err := runCommandBuffers(t, cmd) - if err != nil { - t.Fatalf("bd %s failed: %v\nstdout:\n%s\nstderr:\n%s", - strings.Join(args, " "), err, stdout.String(), stderr.String()) - } - var issues []types.IssueWithCounts - if err := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &issues); err != nil { - t.Fatalf("parse JSON: %v\n%s", err, stdout.String()) - } - ids := make([]string, 0, len(issues)) - for _, issue := range issues { - ids = append(ids, issue.ID) - } - return ids, stdout.String() - } - - t.Run("ready_label_any", func(t *testing.T) { - laneA := bdCreate(t, bd, dir, "Lane A work", "--type", "task", "--label", "lany:lane-a") - bdCreate(t, bd, dir, "Lane B work", "--type", "task", "--label", "lany:lane-b") - laneC := bdCreate(t, bd, dir, "Lane C work", "--type", "task", "--label", "lany:lane-c") - - ids, raw := readyIDs(t, "ready", "--json", "--label-any", "lany:lane-a,lany:lane-c") - - want := map[string]bool{laneA.ID: true, laneC.ID: true} - if len(ids) != len(want) { - t.Fatalf("--label-any returned %d issues, want %d — an unfiltered ready set means the OR-set clause was dropped: %s", - len(ids), len(want), raw) - } - for _, id := range ids { - if !want[id] { - t.Fatalf("--label-any returned %s, which carries neither requested label: %s", id, raw) - } - } - }) - - t.Run("ready_label_any_intersects_with_label", func(t *testing.T) { - both := bdCreate(t, bd, dir, "Tiered lane A work", "--type", "task", - "--label", "lmix:tier", "--label", "lmix:lane-a") - bdCreate(t, bd, dir, "Untiered lane A work", "--type", "task", "--label", "lmix:lane-a") - bdCreate(t, bd, dir, "Tiered lane B work", "--type", "task", - "--label", "lmix:tier", "--label", "lmix:lane-b") - - ids, raw := readyIDs(t, "ready", "--json", - "--label", "lmix:tier", "--label-any", "lmix:lane-a,lmix:lane-c") - - if len(ids) != 1 || ids[0] != both.ID { - t.Fatalf("--label with --label-any returned %v, want only %s (AND-set and OR-set must both apply): %s", - ids, both.ID, raw) - } - }) - - t.Run("ready_claim_label_any_fences_to_its_lane", func(t *testing.T) { - // An exhausted lane must claim nothing. If --label-any is dropped here - // the claim silently takes unfenced work while the caller believes it - // is fenced to its own lane. - bdCreate(t, bd, dir, "Fence lane B work", "--type", "task", "--label", "lfence:lane-b") - - ids, raw := readyIDs(t, "ready", "--claim", "--json", "--label-any", "lfence:lane-a") - - if len(ids) != 0 { - t.Fatalf("--claim --label-any on an empty lane claimed %v, want nothing: %s", ids, raw) - } - }) - // ===== -C flag ===== t.Run("ready_with_C_flag", func(t *testing.T) { @@ -424,6 +305,134 @@ func TestEmbeddedReady(t *testing.T) { }) } +func TestEmbeddedReadyBlockersAndLabels(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "rd") + bdCreate(t, bd, dir, "Ready test issue", "--type", "task") + + // ===== With Blockers ===== + + t.Run("ready_excludes_blocked", func(t *testing.T) { + blocker := bdCreate(t, bd, dir, "Blocker issue", "--type", "task") + blocked := bdCreate(t, bd, dir, "Blocked by blocker", "--type", "task") + + // Add blocking dependency: blocked depends on blocker + cmd := exec.Command(bd, "dep", "add", blocked.ID, blocker.ID) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dep add failed: %v\n%s", err, out) + } + + cmd = exec.Command(bd, "ready") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd ready failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + // The blocked issue should not appear in ready output + if strings.Contains(stdout.String(), "Blocked by blocker") { + t.Errorf("blocked issue should not appear in ready output: %s", stdout.String()) + } + }) + + // ===== Exclude Label ===== + + t.Run("ready_exclude_label", func(t *testing.T) { + bdCreate(t, bd, dir, "Triage pending item", "--type", "task", "--label", "triage:pending") + bdCreate(t, bd, dir, "Normal ready item", "--type", "task") + + cmd := exec.Command(bd, "ready", "--exclude-label", "triage:pending") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd ready --exclude-label failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + if strings.Contains(stdout.String(), "Triage pending item") { + t.Errorf("triage:pending issue should not appear with --exclude-label: %s", stdout.String()) + } + if !strings.Contains(stdout.String(), "Normal ready item") { + t.Errorf("normal issue should still appear with --exclude-label: %s", stdout.String()) + } + }) + + readyIDs := func(t *testing.T, args ...string) ([]string, string) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd %s failed: %v\nstdout:\n%s\nstderr:\n%s", + strings.Join(args, " "), err, stdout.String(), stderr.String()) + } + var issues []types.IssueWithCounts + if err := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &issues); err != nil { + t.Fatalf("parse JSON: %v\n%s", err, stdout.String()) + } + ids := make([]string, 0, len(issues)) + for _, issue := range issues { + ids = append(ids, issue.ID) + } + return ids, stdout.String() + } + + t.Run("ready_label_any", func(t *testing.T) { + laneA := bdCreate(t, bd, dir, "Lane A work", "--type", "task", "--label", "lany:lane-a") + bdCreate(t, bd, dir, "Lane B work", "--type", "task", "--label", "lany:lane-b") + laneC := bdCreate(t, bd, dir, "Lane C work", "--type", "task", "--label", "lany:lane-c") + + ids, raw := readyIDs(t, "ready", "--json", "--label-any", "lany:lane-a,lany:lane-c") + + want := map[string]bool{laneA.ID: true, laneC.ID: true} + if len(ids) != len(want) { + t.Fatalf("--label-any returned %d issues, want %d — an unfiltered ready set means the OR-set clause was dropped: %s", + len(ids), len(want), raw) + } + for _, id := range ids { + if !want[id] { + t.Fatalf("--label-any returned %s, which carries neither requested label: %s", id, raw) + } + } + }) + + t.Run("ready_label_any_intersects_with_label", func(t *testing.T) { + both := bdCreate(t, bd, dir, "Tiered lane A work", "--type", "task", + "--label", "lmix:tier", "--label", "lmix:lane-a") + bdCreate(t, bd, dir, "Untiered lane A work", "--type", "task", "--label", "lmix:lane-a") + bdCreate(t, bd, dir, "Tiered lane B work", "--type", "task", + "--label", "lmix:tier", "--label", "lmix:lane-b") + + ids, raw := readyIDs(t, "ready", "--json", + "--label", "lmix:tier", "--label-any", "lmix:lane-a,lmix:lane-c") + + if len(ids) != 1 || ids[0] != both.ID { + t.Fatalf("--label with --label-any returned %v, want only %s (AND-set and OR-set must both apply): %s", + ids, both.ID, raw) + } + }) + + t.Run("ready_claim_label_any_fences_to_its_lane", func(t *testing.T) { + // An exhausted lane must claim nothing. If --label-any is dropped here + // the claim silently takes unfenced work while the caller believes it + // is fenced to its own lane. + bdCreate(t, bd, dir, "Fence lane B work", "--type", "task", "--label", "lfence:lane-b") + + ids, raw := readyIDs(t, "ready", "--claim", "--json", "--label-any", "lfence:lane-a") + + if len(ids) != 0 { + t.Fatalf("--claim --label-any on an empty lane claimed %v, want nothing: %s", ids, raw) + } + }) +} + func TestEmbeddedReadyConcurrent(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") diff --git a/cmd/bd/search_embedded_test.go b/cmd/bd/search_embedded_test.go index ebc30af45..fae6d0808 100644 --- a/cmd/bd/search_embedded_test.go +++ b/cmd/bd/search_embedded_test.go @@ -64,7 +64,7 @@ func bdSearchJSON(t *testing.T, bd, dir string, args ...string) []map[string]int return results } -func TestEmbeddedSearch(t *testing.T) { +func TestEmbeddedSearchQueryStatusAssigneeLabels(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -140,42 +140,6 @@ func TestEmbeddedSearch(t *testing.T) { } }) - t.Run("search_status_all", func(t *testing.T) { - // Use prefix to match all issues - results := bdSearchJSON(t, bd, dir, "sr-", "--status", "all") - ids := map[string]bool{} - for _, r := range results { - ids[r["id"].(string)] = true - } - if !ids[taskA.ID] || !ids[closedTask.ID] { - t.Error("expected both open and closed issues with --status all") - } - }) - - t.Run("search_default_includes_closed", func(t *testing.T) { - // bd-t5yex: "was this already filed/fixed?" is the dominant search - // query, so the default must not silently hide closed issues. - results := bdSearchJSON(t, bd, dir, "sr-") - ids := map[string]bool{} - for _, r := range results { - ids[r["id"].(string)] = true - } - if !ids[taskA.ID] || !ids[closedTask.ID] { - t.Error("expected default search to include open and closed issues") - } - }) - - t.Run("search_status_comma_separated", func(t *testing.T) { - results := bdSearchJSON(t, bd, dir, "sr-", "--status", "open,closed") - ids := map[string]bool{} - for _, r := range results { - ids[r["id"].(string)] = true - } - if !ids[taskA.ID] || !ids[closedTask.ID] { - t.Error("expected both open and closed issues with --status open,closed") - } - }) - t.Run("search_status_invalid", func(t *testing.T) { out := bdSearchFail(t, bd, dir, "sr-", "--status", "not-a-status") if !strings.Contains(out, "invalid status") || !strings.Contains(out, "not-a-status") { @@ -257,6 +221,97 @@ func TestEmbeddedSearch(t *testing.T) { } }) + // ===== Description Filters ===== + + t.Run("search_desc_contains", func(t *testing.T) { + results := bdSearchJSON(t, bd, dir, "sr-", "--desc-contains", "alpha") + found := false + for _, r := range results { + if r["id"] == taskA.ID { + found = true + } + } + if !found { + t.Error("expected to find taskA with --desc-contains alpha") + } + }) + + t.Run("search_empty_description", func(t *testing.T) { + results := bdSearchJSON(t, bd, dir, "sr-", "--empty-description") + found := false + for _, r := range results { + if r["id"] == taskD.ID { + found = true + } + } + if !found { + t.Error("expected to find task without description") + } + for _, r := range results { + if r["id"] == taskA.ID { + t.Error("task with description should not appear with --empty-description") + } + } + }) + + _ = taskB + _ = taskC + _ = taskD +} + +func TestEmbeddedSearchStatusLimitSortDatesMetadata(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "sr") + + // Create test fixtures + taskA := bdCreate(t, bd, dir, "Alpha task", "--type", "task", "--priority", "1", "--assignee", "alice", "--description", "Important alpha work", "--label", "urgent") + taskB := bdCreate(t, bd, dir, "Beta bug", "--type", "bug", "--priority", "3", "--assignee", "bob", "--description", "Beta bug description", "--label", "backend") + taskC := bdCreate(t, bd, dir, "Gamma feature", "--type", "feature", "--priority", "2", "--label", "urgent", "--label", "frontend") + taskD := bdCreate(t, bd, dir, "Delta task no desc", "--type", "task") + closedTask := bdCreate(t, bd, dir, "Closed epsilon", "--type", "task") + bdClose(t, bd, dir, closedTask.ID) + + t.Run("search_status_all", func(t *testing.T) { + // Use prefix to match all issues + results := bdSearchJSON(t, bd, dir, "sr-", "--status", "all") + ids := map[string]bool{} + for _, r := range results { + ids[r["id"].(string)] = true + } + if !ids[taskA.ID] || !ids[closedTask.ID] { + t.Error("expected both open and closed issues with --status all") + } + }) + + t.Run("search_default_includes_closed", func(t *testing.T) { + // bd-t5yex: "was this already filed/fixed?" is the dominant search + // query, so the default must not silently hide closed issues. + results := bdSearchJSON(t, bd, dir, "sr-") + ids := map[string]bool{} + for _, r := range results { + ids[r["id"].(string)] = true + } + if !ids[taskA.ID] || !ids[closedTask.ID] { + t.Error("expected default search to include open and closed issues") + } + }) + + t.Run("search_status_comma_separated", func(t *testing.T) { + results := bdSearchJSON(t, bd, dir, "sr-", "--status", "open,closed") + ids := map[string]bool{} + for _, r := range results { + ids[r["id"].(string)] = true + } + if !ids[taskA.ID] || !ids[closedTask.ID] { + t.Error("expected both open and closed issues with --status open,closed") + } + }) + // ===== Limit ===== t.Run("search_limit", func(t *testing.T) { @@ -323,39 +378,6 @@ func TestEmbeddedSearch(t *testing.T) { } }) - // ===== Description Filters ===== - - t.Run("search_desc_contains", func(t *testing.T) { - results := bdSearchJSON(t, bd, dir, "sr-", "--desc-contains", "alpha") - found := false - for _, r := range results { - if r["id"] == taskA.ID { - found = true - } - } - if !found { - t.Error("expected to find taskA with --desc-contains alpha") - } - }) - - t.Run("search_empty_description", func(t *testing.T) { - results := bdSearchJSON(t, bd, dir, "sr-", "--empty-description") - found := false - for _, r := range results { - if r["id"] == taskD.ID { - found = true - } - } - if !found { - t.Error("expected to find task without description") - } - for _, r := range results { - if r["id"] == taskA.ID { - t.Error("task with description should not appear with --empty-description") - } - } - }) - // ===== Long Output ===== t.Run("search_long", func(t *testing.T) { diff --git a/cmd/bd/show_embedded_test.go b/cmd/bd/show_embedded_test.go index 04f9649f2..b142aa2f8 100644 --- a/cmd/bd/show_embedded_test.go +++ b/cmd/bd/show_embedded_test.go @@ -300,14 +300,21 @@ func TestEmbeddedShowBasicsAndJSON(t *testing.T) { }) } -func TestEmbeddedShowDetailAndCurrent(t *testing.T) { +// TestEmbeddedShowDetailAndCurrentFlags and +// TestEmbeddedShowDetailAndCurrentHistoryAndCurrent were split from +// TestEmbeddedShowDetailAndCurrent into two top-level tests over disjoint +// subtest groups, each on its own workspace, for CI shard balance. Every +// subtest is preserved exactly once, in its original order within its group; +// show_current_fallback_to_last_touched still follows subtests that show +// issues (show_as_of, show_local_time). +func TestEmbeddedShowDetailAndCurrentFlags(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() bd := buildEmbeddedBD(t) - dir, beadsDir, _ := bdInit(t, bd, "--prefix", "ts") + dir, _, _ := bdInit(t, bd, "--prefix", "ts") // ===== --long ===== @@ -362,6 +369,16 @@ func TestEmbeddedShowDetailAndCurrent(t *testing.T) { // Should not error even with no children _ = bdShowRaw(t, bd, dir, issue.ID, "--children") }) +} + +func TestEmbeddedShowDetailAndCurrentHistoryAndCurrent(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "ts") // ===== --as-of ===== diff --git a/cmd/bd/swarm_embedded_test.go b/cmd/bd/swarm_embedded_test.go index 69379614a..38b812af7 100644 --- a/cmd/bd/swarm_embedded_test.go +++ b/cmd/bd/swarm_embedded_test.go @@ -104,14 +104,22 @@ func createSwarmableEpic(t *testing.T, bd, dir, prefix string) (epicID string, c return epicID, childIDs } -func TestEmbeddedSwarm(t *testing.T) { +// setupEmbeddedSwarm is the TestEmbeddedSwarm* tests' workspace (prefix sw). +func setupEmbeddedSwarm(t *testing.T) (string, string) { + t.Helper() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "sw") + return bd, dir +} + +func TestEmbeddedSwarmValidate(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() - bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "sw") + bd, dir := setupEmbeddedSwarm(t) epicID, _ := createSwarmableEpic(t, bd, dir, "Swarm") @@ -149,6 +157,15 @@ func TestEmbeddedSwarm(t *testing.T) { t.Errorf("expected warning about no children: %s", out) } }) +} + +func TestEmbeddedSwarmCreate(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedSwarm(t) // ===== swarm create ===== @@ -176,6 +193,15 @@ func TestEmbeddedSwarm(t *testing.T) { t.Errorf("expected coordinator=alice, got %v", m["coordinator"]) } }) +} + +func TestEmbeddedSwarmCreateForceAndErrors(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedSwarm(t) t.Run("create_force", func(t *testing.T) { eID, _ := createSwarmableEpic(t, bd, dir, "Force") @@ -202,6 +228,15 @@ func TestEmbeddedSwarm(t *testing.T) { bdSwarm(t, bd, dir, "create", eID) bdSwarmFail(t, bd, dir, "create", eID) }) +} + +func TestEmbeddedSwarmStatusList(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedSwarm(t) // ===== swarm status ===== diff --git a/cmd/bd/unclaim_embedded_test.go b/cmd/bd/unclaim_embedded_test.go index b5535fcd2..d969e6a4f 100644 --- a/cmd/bd/unclaim_embedded_test.go +++ b/cmd/bd/unclaim_embedded_test.go @@ -38,7 +38,7 @@ func bdUnclaimFail(t *testing.T, bd, dir string, args ...string) string { return string(out) } -func TestEmbeddedUnclaim(t *testing.T) { +func TestEmbeddedUnclaimSuccess(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -122,6 +122,16 @@ func TestEmbeddedUnclaim(t *testing.T) { t.Errorf("expected status open after unclaim, got %q", unclaimed[0].Status) } }) +} + +func TestEmbeddedUnclaimBatchAndRefusals(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tu") // ===== Multiple IDs ===== diff --git a/cmd/bd/update_embedded_test.go b/cmd/bd/update_embedded_test.go index 1b1841576..f71511b17 100644 --- a/cmd/bd/update_embedded_test.go +++ b/cmd/bd/update_embedded_test.go @@ -243,12 +243,14 @@ func TestEmbeddedUpdateRoutedStoreCommitsTargetHead(t *testing.T) { } } -// TestEmbeddedUpdateFields and TestEmbeddedUpdateLifecycle were split from -// TestEmbeddedUpdate (originally ~267s, measured under --config=embedded) -// into 2 top-level tests over disjoint subtest groups, for CI shard balance -// (see scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedUpdateFields(t *testing.T) { +// The TestEmbeddedUpdateFields* and TestEmbeddedUpdateLifecycle* tests were +// split from TestEmbeddedUpdate (originally ~267s, measured under +// --config=embedded), first into TestEmbeddedUpdateFields and +// TestEmbeddedUpdateLifecycle and then (~75s each on rbe-west) into these 5 +// top-level tests over disjoint subtest groups, each with its own workspace, +// for CI shard balance (see scripts/ci/embedded_cmd_test_durations.json and +// engdocs/TESTING.md). Every original subtest is preserved exactly once. +func TestEmbeddedUpdateFieldsMappingAndRefs(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -406,6 +408,16 @@ func TestEmbeddedUpdateFields(t *testing.T) { t.Errorf("expected external_ref field to be omitted from JSON after clear, got: %s", rawA) } }) +} + +func TestEmbeddedUpdateFieldsNotesOverwrite(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tu") t.Run("update_notes_overwrite_requires_force", func(t *testing.T) { issue := bdCreate(t, bd, dir, "Notes force test", "--type", "task") @@ -466,6 +478,16 @@ func TestEmbeddedUpdateFields(t *testing.T) { t.Errorf("expected notes %q, got %q", "replacement notes", got.Notes) } }) +} + +func TestEmbeddedUpdateFieldsNotesClearAndDefer(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tu") t.Run("update_empty_notes_refused", func(t *testing.T) { // GH#6021: `--notes ""` is what a dead command substitution collapses @@ -534,7 +556,7 @@ func TestEmbeddedUpdateFields(t *testing.T) { }) } -func TestEmbeddedUpdateLifecycle(t *testing.T) { +func TestEmbeddedUpdateLifecycleDefer(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -624,6 +646,16 @@ func TestEmbeddedUpdateLifecycle(t *testing.T) { t.Errorf("expected status=in_progress to be preserved, got %q", got.Status) } }) +} + +func TestEmbeddedUpdateLifecycleClaimParentCommit(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tu") t.Run("update_claim_already_claimed", func(t *testing.T) { issue := bdCreate(t, bd, dir, "Claim fail test", "--type", "task") diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 68b9aeee6..d6e4e1c61 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1037,9 +1037,11 @@ Required` requires them to have run remotely and passed. - The `args` and `env` of every target tagged `embedded`, `dolt-server-proxied` or `dolt-server-integration` are pinned. - `tools/bazel/check_shard_coverage.py` runs after each tier. It requires: - - every Bazel shard of `//cmd/bd:bd_embedded_test` (50; the manifest's - frozen 20-shard block was the retired `test-embedded-cmd` jobs' split, - F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; + - every Bazel shard of `//cmd/bd:bd_embedded_test` and + `//cmd/bd:bd_embedded_part2_test` (50 each: shards 1-50 and 51-100 of + the manifest's 100-shard block, which their ranges must tile exactly + once; the manifest's frozen 20-shard block was the retired + `test-embedded-cmd` jobs' split, F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; the frozen 5-shard block was `test-embedded-storage`'s, F1), `//cmd/bd:bd_proxied_test` (34; the frozen 15-shard block was `test-proxied-cmd`'s, F2) and diff --git a/scripts/ci/embedded_cmd_test_durations.json b/scripts/ci/embedded_cmd_test_durations.json index e5fcfe221..aed2d56d1 100644 --- a/scripts/ci/embedded_cmd_test_durations.json +++ b/scripts/ci/embedded_cmd_test_durations.json @@ -1,19 +1,16 @@ { "_comment": [ "Measured on rbe-west, the lane's own remote executors (instance oss), on", - "2026-10-09: two `bazel test --config=embedded --nocache_test_results", - "//cmd/bd:bd_embedded_test` runs (race, `-test.parallel=4`, 50 shards), so", - "every top-level TestEmbedded* function ran exactly once per run. Each value", - "is the mean of the two runs (the two split", - "TestEmbeddedInitSelectedExcludeRouting{Stealth,Fork} functions ran only in", - "the second). The runs were taken after bdInit started seeding each", - "workspace's embedded database from a per-process schema template", - "(cmd/bd/embedded_init_template_test.go) and the bd subprocesses stopped", - "sleeping 1s at exit (GORACE=atexit_sleep_ms=0 on the target): the sum of", - "test-seconds fell from ~20.2k to ~10.3k on the same executors. Individual", - "tests varied up to ~2x between the two runs with executor load; the sum", - "moved ~4%. The previous capture was local sandboxed execution and", - "correlated only ~0.4 with rbe-west shard times.", + "2026-10-09: one `bazel test --config=embedded --nocache_test_results", + "//cmd/bd:bd_embedded_test //cmd/bd:bd_embedded_part2_test` run (race,", + "`-test.parallel=4`, the 100-shard block split 50+50 over the two targets),", + "so every top-level TestEmbedded* function ran exactly once. It was taken", + "after the slowest functions were split into smaller top-level functions", + "(subtests moved verbatim): the sum of test-seconds fell from ~10.3k", + "(50 shards, the previous capture) to ~8.1k, partly because each shard now", + "runs fewer tests at once. The remaining >60s functions are the *Concurrent", + "stress tests, whose bd subprocesses queue on the workspace's single writer", + "lock, and fixtures that seed many issues one `bd create` at a time.", "", "Each value is that top-level test's own from Bazel's", "generated test.xml, NOT the enclosing : 0 && p.total != parts[0].total) { + t.Fatalf("cmd/bd:%s runs shards %d..%d of %d; want it to start at shard %d of the same block as %v", name, p.offset+1, p.offset+p.shards, p.total, next+1, parts) + } + next += p.shards + parts = append(parts, p) + } + if next != parts[0].total { + t.Fatalf("%v run %d shards of a %d-shard block; their shard_counts must sum to --shard-total", parts, next, parts[0].total) + } + return parts +} + +// bazelEmbeddedCmdShardCount returns the shard total of the bazel-embedded +// lane's cmd/bd manifest block, which embeddedCmdTargets split between them +// (bazelEmbeddedCmdParts): the single source of truth for the Bazel-only +// lane's cmd/bd shard split (slice F1), which no longer has to equal PR +// Risk's/main.yml's legacy "Test (Embedded Dolt Cmd N/20)" fork/push jobs' +// matrix size — mirrors bazelProxiedShardCount (F2; see that function's doc +// comment below for the shared rationale, not repeated here). func bazelEmbeddedCmdShardCount(t *testing.T) int { t.Helper() - root := sourceRepoRoot(t) - rule := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_embedded_test") - m := shardCountPattern.FindStringSubmatch(rule) - if m == nil { - t.Fatalf("cmd/bd:bd_embedded_test has no `shard_count = N,` in cmd/bd/BUILD.bazel:\n%s", rule) - } - n, err := strconv.Atoi(m[1]) - if err != nil { - t.Fatalf("cmd/bd:bd_embedded_test shard_count: %v", err) - } - return n + return bazelEmbeddedCmdParts(t)[0].total } // bazelEmbeddedStorageShardCount returns diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 52ff1b463..d1e800aec 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -322,7 +322,6 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { // the shared rationale. liveShardCount := map[string]func(*testing.T) int{ "//cmd/bd:bd_proxied_test": bazelProxiedShardCount, - "//cmd/bd:bd_embedded_test": bazelEmbeddedCmdShardCount, "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": bazelEmbeddedStorageShardCount, "//internal/storage/dolt:dolt_server_full_test": bazelServerFullShardCount, } @@ -342,7 +341,8 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { // check_shard_coverage.py arguments, not independently // hard-coded literals that could drift from BUILD.bazel // unnoticed (S1). - {"test-embedded-cmd", "Test", "//cmd/bd:bd_embedded_test", ".github/scripts/embedded-test-shard.sh", -1}, + // The cmd/bd block is split over embeddedCmdTargets: one + // --suite each, SHARDS@OFFSET/TOTAL (added below). {"test-embedded-storage", "Test", "//internal/storage/embeddeddolt:embeddeddolt_embedded_test", ".github/scripts/embedded-storage-test-shard.sh", -1}, }, []string{"//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test", "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test"}}, {bazelProxiedJobName, "doltserver-proxied", []suite{ @@ -365,6 +365,11 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { job := readCIWorkflow(t, bazelWorkflowName).job(t, c.lane) step := job.step(t, "Every listed test ran in its shard") want := []string{"python3 tools/bazel/check_shard_coverage.py", `--bep "$RUNNER_TEMP/bazel-bep.json"`} + if c.lane == bazelEmbedJobName { + for _, p := range bazelEmbeddedCmdParts(t) { + want = append(want, fmt.Sprintf("--suite %s %s %s", p.label, ".github/scripts/embedded-test-shard.sh", p.spec())) + } + } for _, s := range c.suites { shards := s.shardCount if shards < 0 { @@ -531,6 +536,51 @@ esac t.Errorf("--whole %v: pass = %v, want %v (mention %q):\n%s", c.names, err == nil, c.pass, c.mention, out) } } + // A 2-shard block split over two 1-shard targets (SHARDS@OFFSET/TOTAL): + // //pkg:p's shard 1 is the script's shard 1, //pkg:q's shard 1 its shard + // 2. The targets must tile the block exactly once. + splitLogs := filepath.Join(dir, "split") + for target, names := range map[string][]string{"p": {"TestA", "TestB"}, "q": {"TestC"}} { + d := filepath.Join(splitLogs, "pkg", target) + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + xml := `` + for _, n := range names { + xml += `` + } + if err := os.WriteFile(filepath.Join(d, "test.xml"), []byte(xml+``), 0o644); err != nil { + t.Fatal(err) + } + } + splitBEP := filepath.Join(dir, "split.json") + if err := os.WriteFile(splitBEP, []byte(`{"id":{"targetConfigured":{"label":"//pkg:p"}},"configured":{"targetKind":"sh_test rule"}} +{"id":{"testResult":{"label":"//pkg:p","run":1,"attempt":1}}} +{"id":{"targetConfigured":{"label":"//pkg:q"}},"configured":{"targetKind":"sh_test rule"}} +{"id":{"testResult":{"label":"//pkg:q","run":1,"attempt":1}}} +`), 0o644); err != nil { + t.Fatal(err) + } + for _, c := range []struct { + name string + suites []string + pass bool + mention string + }{ + {"split exact", []string{"//pkg:p", "1@0/2", "//pkg:q", "1@1/2"}, true, ""}, + {"split shard run twice", []string{"//pkg:p", "1@0/2", "//pkg:q", "1@0/2"}, false, "run by more than one target"}, + {"split shard run by no target", []string{"//pkg:p", "1@0/2"}, false, "no target runs shard(s) [2]"}, + {"split range past the total", []string{"//pkg:p", "1@0/2", "//pkg:q", "1@2/2"}, false, "exceed"}, + } { + args := []string{script, "--bep", splitBEP, "--testlogs", splitLogs} + for i := 0; i < len(c.suites); i += 2 { + args = append(args, "--suite", c.suites[i], shard, c.suites[i+1]) + } + out, err := exec.Command(python, args...).CombinedOutput() + if (err == nil) != c.pass || (c.mention != "" && !strings.Contains(string(out), c.mention)) { + t.Errorf("%s: pass = %v, want %v (mention %q):\n%s", c.name, err == nil, c.pass, c.mention, out) + } + } if out, err := exec.Command(python, script, "--bep", writeBEP(2), "--testlogs", filepath.Join(dir, "logs0"), "--suite", label, shard, "2", "--whole", "//pkg:absent").CombinedOutput(); err == nil { t.Errorf("--whole for a target the BEP lacks passed:\n%s", out) } @@ -803,15 +853,20 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { args []string env map[string]string } + embeddedCmdEnv := map[string]string{ + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + // bdInit's schema template and the race runtime's exit sleep: + // neither selects or skips a test. + "BEADS_TEST_EMBEDDED_SCHEMA_TOOL": "$(rlocationpath //internal/storage/embeddeddolt/cmd:cmd_norace)", "GORACE": "atexit_sleep_ms=0", + } want := map[string]target{ "//cmd/bd:bd_embedded_test": { - []string{"$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", "-test.timeout=19m"}, - map[string]string{ - "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", - // bdInit's schema template and the race runtime's exit sleep: - // neither selects or skips a test. - "BEADS_TEST_EMBEDDED_SCHEMA_TOOL": "$(rlocationpath //internal/storage/embeddeddolt/cmd:cmd_norace)", "GORACE": "atexit_sleep_ms=0", - }, + []string{"--shard-offset=0", "--shard-total=100", "$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", "-test.timeout=19m"}, + embeddedCmdEnv, + }, + "//cmd/bd:bd_embedded_part2_test": { + []string{"--shard-offset=50", "--shard-total=100", "$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", "-test.timeout=19m"}, + embeddedCmdEnv, }, "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": { []string{"$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)", "-test.timeout=19m"}, diff --git a/tools/bazel/check_shard_coverage.py b/tools/bazel/check_shard_coverage.py index 0e574d348..c088c6e9a 100755 --- a/tools/bazel/check_shard_coverage.py +++ b/tools/bazel/check_shard_coverage.py @@ -2,7 +2,7 @@ """Fail unless every Bazel shard ran exactly the tests its shard script lists. Usage: check_shard_coverage.py --bep [--testlogs DIR] - --suite LABEL SCRIPT SHARDS [--suite ...] + --suite LABEL SCRIPT SHARDS[@OFFSET/TOTAL] [--suite ...] [--whole LABEL ...] The manifest-sharded targets (//cmd/bd:bd_embedded_test runs @@ -24,6 +24,13 @@ BEADS_TEST_SKIP or a lost BEADS_TEST_EMBEDDED_DOLT=1 turns the tier into t.Skip calls, which still list every test. It needs test.xml locally and -test.v in it, as --config=embedded sets. +A manifest block bigger than Bazel's 50-shard cap per rule runs as several +targets (tools/bazel/go_test_manifest_shard.sh --shard-offset/--shard-total): +SHARDS@OFFSET/TOTAL says Bazel shard k of LABEL is the script's shard +OFFSET+k of TOTAL. The suites sharing a SCRIPT and TOTAL must cover shards +1..TOTAL exactly once between them, so (the script assigning each test to +one shard of TOTAL) every test runs in exactly one shard of one target. + Exit status: 0 if every shard matches, 1 otherwise. """ @@ -39,6 +46,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from equivalence import read_bep, testlog_xmls # noqa: E402 LISTED = re.compile(r"^ (Test[A-Za-z0-9_]*)$") +SHARD_SPEC = re.compile(r"^(\d+)(?:@(\d+)/(\d+))?$") # Names the shard scripts' `grep '^func Test'` discovery lists that go test # never runs as a test: TestMain(m *testing.M) is the package's test entry @@ -79,10 +87,36 @@ def all_skipped_problem(where, got, skipped): return None +def tiling_problems(suites): + """Problems with suites of (label, script, shards, offset, total) that + split one SCRIPT's TOTAL-shard block over several targets: together they + must cover shards 1..TOTAL exactly once.""" + problems = [] + parts = {} + for label, script, shards, offset, total in suites: + if offset + shards > total: + problems.append(f"{label}: shards {offset + 1}..{offset + shards} exceed {script}'s total {total}") + parts.setdefault((script, total), []).append((offset, shards, label)) + for (script, total), ranges in sorted(parts.items()): + covered = {} + for offset, shards, label in ranges: + for k in range(offset + 1, offset + shards + 1): + covered.setdefault(k, []).append(label) + missing = [k for k in range(1, total + 1) if k not in covered] + twice = sorted(k for k, labels in covered.items() if len(labels) > 1) + if missing: + problems.append(f"{script} {total}: no target runs shard(s) {missing}") + if twice: + problems.append(f"{script} {total}: shard(s) {twice} run by more than one target") + return problems + + def check(tested, testlogs, suites, lister=listed_tests, whole=()): """Return ([summary lines], [problems]) for suites of (label, script, shards) - and whole (unsharded or sharded) labels that must not be all-skipped.""" - lines, problems = [], [] + or (label, script, shards, offset, total), and whole (unsharded or + sharded) labels that must not be all-skipped.""" + suites = [s if len(s) == 5 else (*s, 0, s[2]) for s in suites] + lines, problems = [], tiling_problems(suites) for label in whole: if label not in tested: problems.append(f"{label}: --whole, but the BEP has no result for it") @@ -103,7 +137,7 @@ def check(tested, testlogs, suites, lister=listed_tests, whole=()): if p: problems.append(p) lines.append(f"{label}: not all skipped") - for label, script, shards in suites: + for label, script, shards, offset, total_shards in suites: if tested.get(label) != shards: problems.append(f"{label}: the BEP has {tested.get(label, 0)} shard(s), want {shards} ({script})") continue @@ -111,12 +145,12 @@ def check(tested, testlogs, suites, lister=listed_tests, whole=()): # The scripts' hash fallback forks per test (seconds per shard for # the server suite's ~1200 tests): list the shards concurrently. with concurrent.futures.ThreadPoolExecutor(max_workers=min(8, os.cpu_count() or 2)) as pool: - listings = [pool.submit(lister, script, k, shards) for k in range(1, shards + 1)] + listings = [pool.submit(lister, script, offset + k, total_shards) for k in range(1, shards + 1)] for k, xml_path in enumerate(testlog_xmls(testlogs, label, shards), start=1): try: want = listings[k - 1].result() except (OSError, subprocess.CalledProcessError) as e: - problems.append(f"{label}: {script} {k} {shards} failed: {e}") + problems.append(f"{label}: {script} {offset + k} {total_shards} failed: {e}") continue rel = os.path.relpath(xml_path, testlogs) if not os.path.exists(xml_path): @@ -136,7 +170,8 @@ def check(tested, testlogs, suites, lister=listed_tests, whole=()): f"(not in the target's srcs?)") for name in sorted(got - want): problems.append(f"{label} shard {k}/{shards}: {name} ran but {script} does not list it there") - lines.append(f"{label}: {shards} shards, {total} listed tests") + span = "" if (offset, total_shards) == (0, shards) else f" (script shards {offset + 1}..{offset + shards} of {total_shards})" + lines.append(f"{label}: {shards} shards{span}, {total} listed tests") return lines, problems @@ -145,16 +180,21 @@ def main(argv=None): ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") ap.add_argument("--testlogs", default=None, help="default: from the BEP, else ./bazel-testlogs") ap.add_argument("--suite", nargs=3, action="append", required=True, metavar=("LABEL", "SCRIPT", "SHARDS"), - help="a manifest-sharded target, its shard script and shard count (repeatable)") + help="a manifest-sharded target, its shard script and shard count, as N or " + "N@OFFSET/TOTAL for one of several targets splitting a TOTAL-shard block (repeatable)") ap.add_argument("--whole", action="append", default=[], metavar="LABEL", help="a target none of whose test.xml may be all skipped (repeatable)") args = ap.parse_args(argv) suites = [] - for label, script, shards in args.suite: - if not shards.isdigit() or int(shards) < 1: - ap.error(f"--suite {label}: SHARDS must be a positive integer, got {shards!r}") - suites.append((label, script, int(shards))) + for label, script, spec in args.suite: + m = SHARD_SPEC.match(spec) + if not m or int(m.group(1)) < 1: + ap.error(f"--suite {label}: SHARDS must be N or N@OFFSET/TOTAL with N >= 1, got {spec!r}") + shards = int(m.group(1)) + offset = int(m.group(2)) if m.group(2) else 0 + total = int(m.group(3)) if m.group(3) else shards + suites.append((label, script, shards, offset, total)) tested, _, bep_testlogs = read_bep(args.bep) testlogs = args.testlogs or bep_testlogs or "bazel-testlogs" lines, problems = check(tested, testlogs, suites, whole=args.whole) diff --git a/tools/bazel/go_test_manifest_shard.sh b/tools/bazel/go_test_manifest_shard.sh index 473e53bf3..ed6f7cdc4 100755 --- a/tools/bazel/go_test_manifest_shard.sh +++ b/tools/bazel/go_test_manifest_shard.sh @@ -8,8 +8,8 @@ # go_test_variant.sh it reuses the go_test's binary instead of compiling the # package again. # -# Usage: go_test_manifest_shard.sh -# [binary args...] +# Usage: go_test_manifest_shard.sh [--shard-offset=N --shard-total=M] +# [binary args...] # , : runfiles paths ($(rootpath ...)), relative to # the workspace runfiles directory, which is the test's working directory # and where the script's source globs (cmd/bd/*_embedded_test.go, ...) @@ -22,11 +22,29 @@ # binary flags. # # Bazel's shard k of n (shard_count must equal the manifest's shard total) -# becomes the script's shard k+1 of n. The binary itself must not shard again, +# becomes the script's shard k+1 of n. A manifest block larger than Bazel's +# 50-shard cap per rule is split over several targets: with --shard-offset=N +# --shard-total=M, shard k of n becomes the script's shard N+k+1 of M, so +# targets with offsets 0, n1, n1+n2, ... and shard counts summing to M run +# every shard of the block exactly once (check_shard_coverage.py checks +# that the targets' ranges tile 1..M). The binary itself must not shard again, # so TEST_TOTAL_SHARDS/TEST_SHARD_INDEX are removed after this wrapper tells # Bazel it supports sharding. The go_test's own `env` is not inherited (see # go_test_variant.sh); the target repeats it. set -euo pipefail +offset=0 +part_total="" +while [[ "${1:-}" == --shard-* ]]; do + case "$1" in + --shard-offset=*) offset="${1#*=}" ;; + --shard-total=*) part_total="${1#*=}" ;; + *) + echo "go_test_manifest_shard.sh: unknown flag $1" >&2 + exit 2 + ;; + esac + shift +done script="$1" bin_var="$2" bin="$3" @@ -34,6 +52,17 @@ shift 3 total="${TEST_TOTAL_SHARDS:-1}" index="${TEST_SHARD_INDEX:-0}" +if [[ -n "$part_total" ]]; then + if ! [[ "$offset" =~ ^[0-9]+$ && "$part_total" =~ ^[0-9]+$ ]] || ((offset + total > part_total)); then + echo "go_test_manifest_shard.sh: --shard-offset=$offset with $total shards exceeds --shard-total=$part_total" >&2 + exit 2 + fi + index=$((offset + index)) + total="$part_total" +elif ((offset != 0)); then + echo "go_test_manifest_shard.sh: --shard-offset needs --shard-total" >&2 + exit 2 +fi if [[ -n "${TEST_SHARD_STATUS_FILE:-}" ]]; then touch "$TEST_SHARD_STATUS_FILE" fi From 66075be43ad2631800179ca427f0a8aa0d58eb37 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 21:03:41 +0000 Subject: [PATCH 022/151] ci(bazel): Bazel 9.3.0 (prerequisite for the remote repo contents cache) (#7439) .bazelversion 9.2.0 -> 9.3.0. setup-bazel pins the 9.3.0 release sha256s (bazel-9.3.0-linux-{x86_64,arm64}.sha256) next to 9.2.0's, so the composite stays a valid byte copy for gascity until it bumps too. MODULE.bazel.lock refreshed: 9.3.0's bundled module graph resolves apple_support 2.5.2 and rules_cc 0.2.15 and changes four extension bzlTransitiveDigests. 9.3.0 fixes bazel#30905 (the remote repo contents cache overlay serialized getInputStream), the hard prerequisite for ga-vnycm2.25. Refs ga-vnycm2.23 Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelversion | 2 +- .github/actions/setup-bazel/action.yml | 2 ++ MODULE.bazel.lock | 14 +++++++------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.bazelversion b/.bazelversion index deeb3d66e..b13d146a7 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -9.2.0 +9.3.0 diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index 8bb03ddd1..466316720 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -76,6 +76,8 @@ runs: case "$bazel_version/$arch" in 9.2.0/amd64) bazel_sha=7668a95db1250f12c40407251e4e203b4ec8bf39bc495d2f485b2d8c99048694 ;; 9.2.0/arm64) bazel_sha=049dd21f40ad979db11c3ee68c96a42ce75f1185e69ac61ab20de1501427a410 ;; + 9.3.0/amd64) bazel_sha=d302d22ed77ee5658b87857249baa579db84300ecebe0a1e9a357b5b20f80eb8 ;; + 9.3.0/arm64) bazel_sha=615cc2509819047e4930fe6643c5f01ebd705ccadf75e8f0b85f9b83cc6a6ca0 ;; *) echo "setup-bazel: no pinned sha256 for Bazel '$bazel_version' ($arch); add it from the release's .sha256 file" >&2; exit 1 ;; esac { diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 275b6bfd9..82bd1b0bc 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -18,8 +18,8 @@ "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.21.0/MODULE.bazel": "ac1824ed5edf17dee2fdd4927ada30c9f8c3b520be1b5fd02a5da15bc10bff3e", "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", - "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", - "https://bcr.bazel.build/modules/apple_support/1.24.2/source.json": "2c22c9827093250406c5568da6c54e6fdf0ef06238def3d99c71b12feb057a8d", + "https://bcr.bazel.build/modules/apple_support/2.5.2/MODULE.bazel": "bb1b7081e390a8d3885f642cb0073dfcf8e7a36525e079fa8f5835af7079c0fe", + "https://bcr.bazel.build/modules/apple_support/2.5.2/source.json": "dd51ee925a550e92b55eb41c4c809a967da311a2d2520241252a238a48b39c4c", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/source.json": "ffab9254c65ba945f8369297ad97ca0dec213d3adc6e07877e23a48624a8b456", @@ -138,9 +138,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", + "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", "https://bcr.bazel.build/modules/rules_cc/0.2.17/source.json": "3832f45d145354049137c0090df04629d9c2b5493dc5c2bf46f1834040133a07", - "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", "https://bcr.bazel.build/modules/rules_go/0.63.0/MODULE.bazel": "a2079e783a5d2715a90e6e2970ec8ffd717a487ff2aecb5f553e3975c5294a21", @@ -229,7 +229,7 @@ "moduleExtensions": { "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { "general": { - "bzlTransitiveDigest": "NRXra7941UfmNUyIxnLt82V5hULluVGL2nBsijTl4j4=", + "bzlTransitiveDigest": "BsgERqYptkJeeC8wFaLj251IoTZTSy/58WA6kVp0q8I=", "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", "recordedInputs": [ "REPO_MAPPING:pybind11_bazel+,bazel_tools bazel_tools", @@ -251,7 +251,7 @@ }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { - "bzlTransitiveDigest": "+Kp6j204mBZ3mxlIDDR0gBoP45BZ4jYRhRAcB8sU0qc=", + "bzlTransitiveDigest": "WDuVCwqV73DvcHB5Of+02lhUZgPxr9MqtpvMRyUi0eQ=", "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", "recordedInputs": [ "REPO_MAPPING:rules_kotlin+,bazel_tools bazel_tools" @@ -308,7 +308,7 @@ }, "@@rules_python+//python/extensions:config.bzl%config": { "general": { - "bzlTransitiveDigest": "dzD8Q2YmrP3fz8saWLHPmlwPLO91ImtTmP/c9JKTStM=", + "bzlTransitiveDigest": "L+2ZCpb7GIJqvWaXpzkfk/80d916sIFVmSPzrSm6of8=", "usagesDigest": "ZVSXMAGpD+xzVNPuvF1IoLBkty7TROO0+akMapt1pAg=", "recordedInputs": [ "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", @@ -476,7 +476,7 @@ }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "ijW9KS7qsIY+yBVvJ+Nr1mzwQox09j13DnE3iIwaeTM=", + "bzlTransitiveDigest": "IgkgxNrtlAec7OnaFrWoCC5NQhHcY/aIz4jsqcNMm8Y=", "usagesDigest": "H8dQoNZcoqP+Mu0tHZTi4KHATzvNkM5ePuEqoQdklIU=", "recordedInputs": [ "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", From 72a3d8039d182ce0d77765fe90d567ffc74dce4f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 21:21:20 +0000 Subject: [PATCH 023/151] ci(bazel): embedded storage tier to 40 shards; split its two slowest tests (#7440) //internal/storage/embeddeddolt:embeddeddolt_embedded_test became the embedded lane's floor once the cmd tier was split (CI on #7438: 20 shards, 220s max, 162s median). Its tests are serial, so a shard's wall time is the sum of its tests; measured on rbe-west with the whole lane running (as CI runs it), its 20 shards took 110-161s. - TestCreateBatchFastPathsMatchPerRow_Embedded (111s) runs createbatchequiv's scenarios as two top-level tests (small+depadd, apply+waitsfor) through a new createbatchequiv.RunNamed; the same scenarios and checks. TestCreateBatchEquivalencePartsCoverEveryScenario fails unless the parts name every createbatchequiv.Run scenario exactly once. - TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword (81s, eight cases each opening its own store) runs its table as two top-level tests, cases 1-4 and 5-8; cases and assertions unchanged. - Durations re-measured on rbe-west under the whole lane's load; the Bazel-only block is now 40 shards (LPT heaviest = the heaviest single test, TestLifecycleCloseReopenContract ~71s). One target: 40 is under Bazel's 50-shard cap, so no offset split. rbe-west, `bazel test //... --config=embedded --nocache_test_results`: storage shards max 161s -> 89s, median ~118s -> 63s. Bead: ga-vnycm2.28 Co-authored-by: Claude Opus 5.5 (1M context) --- .../scripts/embedded-storage-test-shards.txt | 823 +++++++++--------- .github/workflows/bazel.yml | 13 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 2 +- .../createbatchequiv/createbatchequiv.go | 32 + internal/storage/embeddeddolt/BUILD.bazel | 6 +- .../create_batch_equivalence_test.go | 44 +- .../federation_credentials_test.go | 18 +- .../ci/embedded_storage_test_durations.json | 760 ++++++++-------- .../ci/gen_embedded_storage_shard_manifest.py | 2 +- scripts/shard_manifest_test.go | 4 +- 10 files changed, 905 insertions(+), 799 deletions(-) diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 0b2f4d259..7e9f2991d 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -114,12 +114,12 @@ # # Format: # -# 20-shard split for the Bazel-only embedded-Dolt storage tier +# 40-shard split for the Bazel-only embedded-Dolt storage tier # (bazel-embedded in .github/workflows/bazel.yml), bin-packed # longest-processing-time-first by measured wall-time from # scripts/ci/embedded_storage_test_durations.json (see that file for provenance # and its "relative weight, not absolute SLA" caveat). -# Regenerate with scripts/ci/gen_embedded_storage_shard_manifest.py 20 +# Regenerate with scripts/ci/gen_embedded_storage_shard_manifest.py 40 # --weights=duration --write after adding or removing Test* functions in # internal/storage/embeddeddolt/*_test.go. --write is incremental: it # keeps every already-assigned test on its current shard and only places @@ -134,420 +134,443 @@ # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator or the durations file instead. -20 1 TestCountsWithMissingWispPlaneTables -20 1 TestCreateBatchFastPathsMatchPerRow_Embedded -20 1 TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween -20 1 TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder -20 1 TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle -20 1 TestEmbeddedIssueOperationsUpdateLabelPatchValueRules -20 1 TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue -20 1 TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates -20 1 TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry -20 1 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor -20 1 TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue -20 1 TestIgnoredCursorHealResumeRespectsOperatorOverride -20 1 TestReadyCounterContract -20 1 TestReclaimRefusesForeignReplicaLease +40 1 TestLifecycleCloseReopenContract -20 2 TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed -20 2 TestEmbeddedConfigCommitStagesConfig -20 2 TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest -20 2 TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent -20 2 TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps -20 2 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering -20 2 TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes -20 2 TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder -20 2 TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne -20 2 TestEmbeddedRelationsRefusesTheZeroDirection -20 2 TestEmbeddedReopenIssueCategorySemantics -20 2 TestEmbeddedTransactionWispLifecyclePersistsDurableDependent -20 2 TestEventsJournalAccessor -20 2 TestGetDependentRecordsEmbedded -20 2 TestLifecycleCloseReopenContract -20 2 TestOpenReturnsCachedStore -20 2 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials +40 2 TestBatchApplyContract -20 3 TestBatchApplyContract -20 3 TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget -20 3 TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate -20 3 TestEmbeddedDependencyEditorRefusalWritesNothing -20 3 TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp -20 3 TestEmbeddedIssueOperationsRequestValuesAreNotMutated -20 3 TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance -20 3 TestEmbeddedReaderDoesNotMutateTheCallerRequest -20 3 TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress -20 3 TestEmbeddedReadyClaimerSkipsIneligibleFrontRows -20 3 TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary -20 3 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff -20 3 TestGetIssueCommentsPageEmbedded -20 3 TestGetReadyWorkIncludeEphemeralAppliesWispFilters -20 3 TestJournalContract -20 3 TestSearchWithMissingWispPlaneTables +40 3 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase +40 3 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase +40 3 TestBuildDSN_NoDatabase +40 3 TestBuildDSN_SpacesInPath +40 3 TestBuildDSN_WithDatabase +40 3 TestCleanGitRemoteCacheGarbage +40 3 TestCleanGitRemoteCacheGarbage_NoCacheDir +40 3 TestCleanGitRemoteCacheGarbage_Throttled +40 3 TestCommitAllOutOfBandChangeAndHonestNoOp +40 3 TestCreateBatchEquivalencePartsCoverEveryScenario +40 3 TestCreateBatchFastPathsMatchPerRowLarge_Embedded +40 3 TestCrossRecordInvariantContract +40 3 TestDepRekeyMarkerStillInIgnoredSeries +40 3 TestDoltCommitResponseLossIsIndeterminate +40 3 TestEmbeddedDoltStoreIssueLifecycleBuildsOperations +40 3 TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore +40 3 TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites +40 3 TestEmbeddedSQLCommitResponseLossIsIndeterminate +40 3 TestEpochContract +40 3 TestExpectedRevisionContract +40 3 TestGitRefPinsMatchDolt +40 3 TestHasRepository +40 3 TestHelperProcess +40 3 TestLargeBatchApplyStatementCounts712_Embedded +40 3 TestLargeGraphCreateTiming_Embedded +40 3 TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance +40 3 TestLenientGateWarningBody_ShapedDecisions +40 3 TestLenientGateWarning_ComposedOutput +40 3 TestMain +40 3 TestPushCommitsPendingChangesFirst +40 3 TestRemoteEntryPointsUseVersionControlOps +40 3 TestRetentionContract +40 3 TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed +40 3 TestTransactionCleanupAfterSQLCommitIsIndeterminate +40 3 TestTransactionCleanupBeforeSQLCommitRemainsDefinite +40 3 TestUnknownProvenanceLeaseStaysReclaimable -20 4 TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes -20 4 TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded -20 4 TestEmbeddedDependencyEditorAttributesItsEventsToTheActor -20 4 TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses -20 4 TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose -20 4 TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge -20 4 TestEmbeddedMergeAndSettleReportsOperatorConflicts -20 4 TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing -20 4 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet -20 4 TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength -20 4 TestEventsSinceEmbedded -20 4 TestIgnoredCursorHealConvergesFromEveryCrashState -20 4 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle -20 4 TestReleaserContract -20 4 TestRemoteGitDataRefOnEngine -20 4 TestUpdateMergeOps_DoltEngine_TwoIssues -20 4 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword +40 4 TestCreateBatchFastPathsMatchPerRowSmallDepAdd_Embedded +40 4 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering +40 4 TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay +40 4 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle -20 5 TestCommitAllOutOfBandChangeAndHonestNoOp -20 5 TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles -20 5 TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure -20 5 TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone -20 5 TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge -20 5 TestEmbeddedDependencyEditorValidationRefusalsWriteNothing -20 5 TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure -20 5 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender -20 5 TestEmbeddedMergeAndSettleMetadataConflict -20 5 TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch -20 5 TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt -20 5 TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary -20 5 TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants -20 5 TestEmbeddedRelationsResolvesAWispAnchor -20 5 TestIgnoredCursorHealNeverResurrectsRolledBackVersions -20 5 TestLifecycleCreateContract -20 5 TestPushCommitsPendingChangesFirst +40 5 TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState +40 5 TestEmbeddedIssueOperationsCloseAndReopenAWisp +40 5 TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder +40 5 TestWithPeerAuth -20 6 TestConcurrencyMultiProcess -20 6 TestConcurrentOpenReturnsCached -20 6 TestDualWriteFixtureKitIsWired -20 6 TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed -20 6 TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt -20 6 TestEmbeddedDependencyEditorAcceptsADiamond -20 6 TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape -20 6 TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone -20 6 TestEmbeddedLenientOpenWarning_DataBehind -20 6 TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder -20 6 TestEmbeddedReaderReadyDeferredAndEphemeralGates -20 6 TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer -20 6 TestEventsJournalRestartAndRetentionBoundary -20 6 TestGraphCounterContract -20 6 TestGraphWalkPlansHonorJoinHints -20 6 TestHistoryPresenceContract -20 6 TestPromoteClearsNoHistory -20 6 TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery +40 6 TestCreateBatchFastPathsMatchPerRowApplyWaitsFor_Embedded +40 6 TestEmbeddedMergeAndSettleFKCascadeRepair +40 6 TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict +40 6 TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates +40 6 TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection +40 6 TestPromoteStillWorksWithCollisionGuard +40 6 TestSearchWithMissingWispPlaneTables -20 7 TestCreateIssuesRejectStaleUpserts -20 7 TestDecryptPassword_KeyMismatchNamesTheLocalKey -20 7 TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex -20 7 TestEmbeddedDependencyEditorSnapshotsTheRequest -20 7 TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn -20 7 TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps -20 7 TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged -20 7 TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn -20 7 TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary -20 7 TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage -20 7 TestEmbeddedRelationsDirectionSelectsTheInverseGraph -20 7 TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether -20 7 TestEmbeddedRelationsResolvesTheAnchorIDExactly -20 7 TestGetDependentRecordsForIssuesEmbedded -20 7 TestHeartbeatRejectsWispEmbedded -20 7 TestPromoteStillWorksWithCollisionGuard -20 7 TestUpdateMergeOps_DoltEngine -20 7 TestWithPeerAuth +40 7 TestConcurrencyMultiProcess +40 7 TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded +40 7 TestEmbeddedDependencyEditorAcceptsAnExternalTarget +40 7 TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked +40 7 TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates +40 7 TestEmbeddedLenientOpenWarning_DataBehind +40 7 TestEmbeddedRelationsLeavesTheCallersRequestAlone +40 7 TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict -20 8 TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting -20 8 TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion -20 8 TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches -20 8 TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState -20 8 TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress -20 8 TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints -20 8 TestEmbeddedReaderListEmptyPageIsWellFormed -20 8 TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix -20 8 TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped -20 8 TestEmbeddedRelationsRefusesAnUnusableTypeFilter -20 8 TestEmbeddedRoleFixtureKitHooksAreUsable -20 8 TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit -20 8 TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges -20 8 TestGetNextChildID -20 8 TestListWisps -20 8 TestPreviewStoreRefusesLocalMetadataWrite -20 8 TestReclaimAnyReplicaOverride -20 8 TestStatsReporterContract -20 8 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials +40 8 TestDeleterContract +40 8 TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane +40 8 TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge +40 8 TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows +40 8 TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether +40 8 TestEmbeddedReopenIssueCategorySemantics +40 8 TestGetIssueCommentsPageEmbedded +40 8 TestListWisps -20 9 TestDeleterContract -20 9 TestDualWriteContractFlagOff -20 9 TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds -20 9 TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies -20 9 TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants -20 9 TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges -20 9 TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest -20 9 TestEmbeddedReaderListMaxRowsIsHonored -20 9 TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse -20 9 TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection -20 9 TestForeignSkipDetailIsBounded -20 9 TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore -20 9 TestLeaseLifecycleEmbedded -20 9 TestOpenAfterCloseCreatesNewStore -20 9 TestPartitionWispIDs -20 9 TestReadyWorkWithMissingDeferredParentTables -20 9 TestRemoteVerbsPresentStoredPeerCredentials -20 9 TestRunInTransactionPostCommitStageFailureIsIndeterminate -20 9 TestWithPeerAuth_MissingRemoteFailsClosed +40 9 TestEmbeddedDependencyEditorRefusesCrossPlaneCycle +40 9 TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent +40 9 TestEmbeddedDependencyEditorSnapshotsTheRequest +40 9 TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation +40 9 TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix +40 9 TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities +40 9 TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps +40 9 TestPristineEmbeddedDoltFixtureRelocatesPrefixes +40 9 TestUpdateMergeOps_DoltEngine -20 10 TestEdgeReaderContract -20 10 TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape -20 10 TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState -20 10 TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes -20 10 TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes -20 10 TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace -20 10 TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel -20 10 TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField -20 10 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers -20 10 TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal -20 10 TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape -20 10 TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows -20 10 TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset -20 10 TestEmbeddedRelationsLeavesTheCallersRequestAlone -20 10 TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing -20 10 TestEventsJournalPruneNeverSplitsASkewedPair -20 10 TestGitRefPinsMatchDolt -20 10 TestIsBlockedBatchEmbedded -20 10 TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance -20 10 TestMain -20 10 TestRetentionContract +40 10 TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween +40 10 TestEmbeddedDependencyEditorRemoveIsIdempotent +40 10 TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix +40 10 TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes +40 10 TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary +40 10 TestGraphCounterContract +40 10 TestIgnoredCursorHealRespectsOperatorOverride +40 10 TestRunInTransactionPostCommitStageFailureIsIndeterminate +40 10 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordAmbient -20 11 TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory -20 11 TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy -20 11 TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex -20 11 TestEmbeddedClaimerRefusalMessagesCarryTheirFragments -20 11 TestEmbeddedClaimerRefusesAWispIDAsNotFound -20 11 TestEmbeddedDependencyEditorRemoveIsIdempotent -20 11 TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion -20 11 TestEmbeddedMigration0058HealsAlreadyAffectedDatabase -20 11 TestEmbeddedMigration0062FlipsEventsToIgnoredPlane -20 11 TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides -20 11 TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters -20 11 TestEmbeddedReaderReadyLimitBoundary -20 11 TestEmbeddedReadyClaimerEmptyFrontIsNormal -20 11 TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps -20 11 TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern -20 11 TestLifecycleUpdateContract -20 11 TestReadyWorkStatementBudget -20 11 TestRemoteVerbPlainRemoteKeepsAmbientEnv -20 11 TestRenameLabelMintsOneVersionPerTouchedBead +40 11 TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges +40 11 TestEmbeddedClaimerRefusalMessagesCarryTheirFragments +40 11 TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches +40 11 TestEmbeddedMergeAndSettleRefusesUnknownTable +40 11 TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset +40 11 TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin +40 11 TestEventsJournal_ActorMatrixOnEmbedded +40 11 TestMemoriesContract +40 11 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordStored -20 12 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase -20 12 TestBuildDSN_SpacesInPath -20 12 TestCleanGitRemoteCacheGarbage_NoCacheDir -20 12 TestCrossRecordInvariantContract -20 12 TestCycleDetectorContract -20 12 TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch -20 12 TestDualWriteContract -20 12 TestEmbeddedBlockedRecheckRecordsUnblockingWrites -20 12 TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded -20 12 TestEmbeddedClaimIssueWrapperClaimsTheWispPlane -20 12 TestEmbeddedCommitToleratesNothingToCommit -20 12 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType -20 12 TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove -20 12 TestEmbeddedDoltStoreIssueLifecycleBuildsOperations -20 12 TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape -20 12 TestEmbeddedIssueOperationsCloseAndReopenAWisp -20 12 TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge -20 12 TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists -20 12 TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities -20 12 TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit -20 12 TestEmbeddedSQLCommitResponseLossIsIndeterminate -20 12 TestEventsAutoPruneBoundsTheJournalToTheFloors -20 12 TestHasRepository -20 12 TestHistory_NullTextColumns -20 12 TestLenientGateWarningBody_ShapedDecisions -20 12 TestMetadataCASContract -20 12 TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed -20 12 TestWorkspaceConfigContract +40 12 TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra +40 12 TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne +40 12 TestEmbeddedClaimerRefusesAWispIDAsNotFound +40 12 TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove +40 12 TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape +40 12 TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID +40 12 TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists +40 12 TestHistoryPresenceContract +40 12 TestLifecycleUpdateContract -20 13 TestAddDependency -20 13 TestAddLabel -20 13 TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing -20 13 TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder -20 13 TestEmbeddedDependencyEditorRefusesAMissingLocalTarget -20 13 TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge -20 13 TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent -20 13 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp -20 13 TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers -20 13 TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables -20 13 TestEmbeddedIssueOperationsCreateAggregatesRelations -20 13 TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing -20 13 TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded -20 13 TestEmbeddedRecomputeAllBlockedWiring -20 13 TestHeartbeatMatchesActorAcrossSpellingEmbedded -20 13 TestLegacyTrackedIgnoredCursorHealsOnOpen -20 13 TestMemoriesContract -20 13 TestQuerierContract -20 13 TestTreeWalkerContract +40 13 TestEmbeddedDepRekeyConvergesRenameChain +40 13 TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges +40 13 TestEmbeddedDependencyEditorRefusesAGhostSource +40 13 TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure +40 13 TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim +40 13 TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent +40 13 TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape +40 13 TestEmbeddedRelationsDirectionSelectsTheInverseGraph +40 13 TestGraphWalkPlansHonorJoinHints -20 14 TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest -20 14 TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed -20 14 TestEmbeddedDepRekeyMergesFreshDuplicateEdges -20 14 TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes -20 14 TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes -20 14 TestEmbeddedDependencyEditorRefusesAGhostSource -20 14 TestEmbeddedDependencyEditorRefusesCrossPlaneCycle -20 14 TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked -20 14 TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants -20 14 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse -20 14 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane -20 14 TestEmbeddedGetMergeBlockersOnAQuietStore -20 14 TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones -20 14 TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict -20 14 TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse -20 14 TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree -20 14 TestGetLabels -20 14 TestPristineEmbeddedDoltFixtureRelocatesPrefixes -20 14 TestSchemaAfterInit -20 14 TestWithPeerAuth_RestoresAbsentEnv +40 14 TestBatchCreatorContract +40 14 TestBlockingAnnotatorContract +40 14 TestCountsIncludeWispDependencies +40 14 TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults +40 14 TestEmbeddedOpenForRemoteSync_DataBehind +40 14 TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage +40 14 TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType +40 14 TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState +40 14 TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest +40 14 TestHistory_NullTextColumns -20 15 TestBuildDSN_NoDatabase -20 15 TestCleanGitRemoteCacheGarbage -20 15 TestCommenterContract -20 15 TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra -20 15 TestCreateBatchFastPathsMatchPerRowLarge_Embedded -20 15 TestDoltCommitResponseLossIsIndeterminate -20 15 TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing -20 15 TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards -20 15 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest -20 15 TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop -20 15 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy -20 15 TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane -20 15 TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies -20 15 TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel -20 15 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext -20 15 TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites -20 15 TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet -20 15 TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType -20 15 TestEmbeddedRenameKeepsDependencyIDsDeterministic -20 15 TestEventsJournalReadRefusesAnInteriorHole -20 15 TestEventsJournal_EmbeddedConcurrentGaplessNoDup -20 15 TestExpectedRevisionContract -20 15 TestLargeBatchApplyStatementCounts356_Embedded -20 15 TestLargeGraphCreateTiming_Embedded -20 15 TestLeaseReclaimerContract -20 15 TestReadyWorkPageTotalFilterMatrix -20 15 TestReclaimExpiredLeaseSurvivesRestartEmbedded -20 15 TestRemoteEntryPointsUseVersionControlOps -20 15 TestTransactionCleanupBeforeSQLCommitRemainsDefinite -20 15 TestUpdateIssueType +40 15 TestCreateIssuesRejectStaleUpserts +40 15 TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch +40 15 TestEmbeddedGetMergeBlockersOnAQuietStore +40 15 TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables +40 15 TestEmbeddedIssueOperationsRequestValuesAreNotMutated +40 15 TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse +40 15 TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary +40 15 TestGetIssue +40 15 TestLegacyTrackedIgnoredCursorHealsOnOpen +40 15 TestReadyWorkPageTotalFilterMatrix -20 16 TestCreateIssues -20 16 TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch -20 16 TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne -20 16 TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded -20 16 TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure -20 16 TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction -20 16 TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates -20 16 TestEmbeddedMergeAndSettleRefusesUnknownTable -20 16 TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation -20 16 TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes -20 16 TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions -20 16 TestEmbeddedTempRebuildIntermediatesNeverReachHistory -20 16 TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded -20 16 TestEventsJournal_ActorMatrixOnEmbedded -20 16 TestIgnoredCursorHealRespectsOperatorOverride -20 16 TestMergeWithStrategy_TheirsResolvesAndCommits -20 16 TestOpenDifferentDirsReturnsDifferentStores -20 16 TestRemoteVerbPlainRemoteWithoutAuthPresentsNone -20 16 TestUnknownProvenanceLeaseStaysReclaimable +40 16 TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles +40 16 TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder +40 16 TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge +40 16 TestEmbeddedReaderDoesNotMutateTheCallerRequest +40 16 TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress +40 16 TestEmbeddedRenameKeepsDependencyIDsDeterministic +40 16 TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree +40 16 TestGetReadyWorkIncludeEphemeralAppliesWispFilters +40 16 TestMergeWithStrategy_OursKeepsLocal +40 16 TestWithPeerAuth_DivergedRemoteURLFailsClosed -20 17 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase -20 17 TestBuildDSN_WithDatabase -20 17 TestCleanGitRemoteCacheGarbage_Throttled -20 17 TestDepRekeyMarkerStillInIgnoredSeries -20 17 TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty -20 17 TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess -20 17 TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit -20 17 TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting -20 17 TestEmbeddedDepRekeyConvergesRenameChain -20 17 TestEmbeddedDependencyEditorAcceptsAnExternalTarget -20 17 TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb -20 17 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy -20 17 TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore -20 17 TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables -20 17 TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain -20 17 TestEmbeddedReaderGetOptionalRowListsAreOffByDefault -20 17 TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse -20 17 TestEmbeddedReaderReadySortPoliciesOrderTheSameRows -20 17 TestEmbeddedRelationsAnswersInThePinnedOrder -20 17 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges -20 17 TestEpochContract -20 17 TestHelperProcess -20 17 TestLegacyTrackedIgnoredCursorUnwedgesPull -20 17 TestLenientGateWarning_ComposedOutput -20 17 TestReadyWorkPageTotal -20 17 TestSweeperContract -20 17 TestTransactionCleanupAfterSQLCommitIsIndeterminate -20 17 TestWithPeerAuth_RestoresEnvWhenCallbackFails +40 17 TestAddDependency +40 17 TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim +40 17 TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion +40 17 TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps +40 17 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender +40 17 TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary +40 17 TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne +40 17 TestHeartbeatMatchesActorAcrossSpellingEmbedded +40 17 TestStatsReporterContract +40 17 TestVersionReconcilerContract -20 18 TestBatchCreatorContract -20 18 TestBlockingAnnotatorContract -20 18 TestCountsIncludeWispDependencies -20 18 TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp -20 18 TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges -20 18 TestEmbeddedBlockedRecheckFailureKeepsTheWrite -20 18 TestEmbeddedCheckedWrappersRouteTheirPreconditions -20 18 TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe -20 18 TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID -20 18 TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim -20 18 TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary -20 18 TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690 -20 18 TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566 -20 18 TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState -20 18 TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin -20 18 TestLargeBatchApplyWallClock_Embedded -20 18 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint -20 18 TestRenameLabel -20 18 TestVersionReconcilerContract +40 18 TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess +40 18 TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes +40 18 TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes +40 18 TestEmbeddedMergeAndSettleMemoryConfigConflict +40 18 TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder +40 18 TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn +40 18 TestMetadataCASContract +40 18 TestOpenDifferentDirsReturnsDifferentStores +40 18 TestPromoteClearsNoHistory +40 18 TestRemoteVerbPlainRemoteWithoutAuthPresentsNone -20 19 TestBatchGetterContract -20 19 TestBootstrapperContract -20 19 TestCreateIssue -20 19 TestCreateRejectsCrossTableIDCollision -20 19 TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim -20 19 TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge -20 19 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers -20 19 TestEmbeddedMergeAndSettleMemoryConfigConflict -20 19 TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566 -20 19 TestEmbeddedOpenForReadOnlyCommand_LenientGate -20 19 TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest -20 19 TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven -20 19 TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet -20 19 TestEmbeddedSyncCommitsConfigBeforeMerge -20 19 TestGetIssue -20 19 TestGetReadyWorkMoleculeFilter -20 19 TestLargeBatchApplyStatementCounts712_Embedded -20 19 TestReopenIssue -20 19 TestWithPeerAuth_DivergedRemoteURLFailsClosed -20 19 TestWithPeerAuth_KeyMismatchFailsClosed +40 19 TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge +40 19 TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel +40 19 TestEmbeddedMigration0062FlipsEventsToIgnoredPlane +40 19 TestEmbeddedReaderListMaxRowsIsHonored +40 19 TestEmbeddedRelationsAnswersInThePinnedOrder +40 19 TestIgnoredCursorHealConvergesFromEveryCrashState +40 19 TestOpenAfterCloseCreatesNewStore +40 19 TestOpenReturnsCachedStore +40 19 TestRemoteVerbsPresentStoredPeerCredentials +40 19 TestRenameLabelMintsOneVersionPerTouchedBead -20 20 TestCounterContract -20 20 TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow -20 20 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped -20 20 TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion -20 20 TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults -20 20 TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix -20 20 TestEmbeddedMergeAndSettleFKCascadeRepair -20 20 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 -20 20 TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke -20 20 TestEmbeddedOpenForRemoteSync_DataBehind -20 20 TestEmbeddedOpenReadOnly_SkipsGateAndMigrations -20 20 TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay -20 20 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits -20 20 TestEmbeddedRemoteMigrateGate_BlocksReopen -20 20 TestEventsAutoPruneThrottleSkipsTheSecondPass -20 20 TestIgnoredCursorHealIsIdempotent -20 20 TestLargeBatchApplyStatementCountsClassic40_Embedded -20 20 TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict -20 20 TestMergeWithStrategy_OursKeepsLocal -20 20 TestPruneEventsBatchDeletesTheLowestSeqsOnly +40 20 TestCreateIssue +40 20 TestEmbeddedCheckedWrappersRouteTheirPreconditions +40 20 TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps +40 20 TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566 +40 20 TestEmbeddedOpenReadOnly_SkipsGateAndMigrations +40 20 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits +40 20 TestEventsJournalRestartAndRetentionBoundary +40 20 TestForeignSkipDetailIsBounded +40 20 TestHeartbeatRejectsWispEmbedded +40 20 TestIgnoredCursorHealNeverResurrectsRolledBackVersions + +40 21 TestCreateRejectsCrossTableIDCollision +40 21 TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards +40 21 TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants +40 21 TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure +40 21 TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain +40 21 TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer +40 21 TestEmbeddedRemoteMigrateGate_BlocksReopen +40 21 TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit +40 21 TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit +40 21 TestLargeBatchApplyWallClock_Embedded + +40 22 TestEdgeReaderContract +40 22 TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded +40 22 TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies +40 22 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped +40 22 TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet +40 22 TestEmbeddedReaderReadyLimitBoundary +40 22 TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded +40 22 TestEventsJournalPruneNeverSplitsASkewedPair +40 22 TestLargeBatchApplyStatementCounts356_Embedded +40 22 TestLeaseLifecycleEmbedded + +40 23 TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes +40 23 TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed +40 23 TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes +40 23 TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest +40 23 TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566 +40 23 TestEmbeddedReaderListEmptyPageIsWellFormed +40 23 TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven +40 23 TestEmbeddedRoleFixtureKitHooksAreUsable +40 23 TestEventsAutoPruneBoundsTheJournalToTheFloors +40 23 TestJournalContract + +40 24 TestBootstrapperContract +40 24 TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex +40 24 TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting +40 24 TestEmbeddedConfigCommitStagesConfig +40 24 TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape +40 24 TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction +40 24 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 +40 24 TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt +40 24 TestIgnoredCursorHealResumeRespectsOperatorOverride +40 24 TestLargeBatchApplyStatementCountsClassic40_Embedded +40 24 TestReopenIssue + +40 25 TestBatchGetterContract +40 25 TestDualWriteFixtureKitIsWired +40 25 TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed +40 25 TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses +40 25 TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet +40 25 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor +40 25 TestGetDependentRecordsForIssuesEmbedded +40 25 TestIgnoredCursorHealIsIdempotent +40 25 TestIsBlockedBatchEmbedded +40 25 TestReadyWorkPageTotal +40 25 TestWithPeerAuth_RestoresAbsentEnv + +40 26 TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp +40 26 TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints +40 26 TestEmbeddedOpenForReadOnlyCommand_LenientGate +40 26 TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters +40 26 TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants +40 26 TestEmbeddedReadyClaimerSkipsIneligibleFrontRows +40 26 TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped +40 26 TestEmbeddedRelationsRefusesTheZeroDirection +40 26 TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore +40 26 TestReclaimAnyReplicaOverride +40 26 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials + +40 27 TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp +40 27 TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing +40 27 TestEmbeddedDependencyEditorAcceptsADiamond +40 27 TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb +40 27 TestEmbeddedMigration0058HealsAlreadyAffectedDatabase +40 27 TestEmbeddedReaderGetOptionalRowListsAreOffByDefault +40 27 TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength +40 27 TestEventsJournalReadRefusesAnInteriorHole +40 27 TestPartitionWispIDs +40 27 TestRemoteGitDataRefOnEngine +40 27 TestSweeperContract + +40 28 TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy +40 28 TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed +40 28 TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow +40 28 TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting +40 28 TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder +40 28 TestEmbeddedIssueOperationsUpdateLabelPatchValueRules +40 28 TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary +40 28 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet +40 28 TestRenameLabel +40 28 TestUpdateIssueType +40 28 TestWithPeerAuth_RestoresEnvWhenCallbackFails + +40 29 TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure +40 29 TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty +40 29 TestEmbeddedDepRekeyMergesFreshDuplicateEdges +40 29 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest +40 29 TestEmbeddedMergeAndSettleMetadataConflict +40 29 TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides +40 29 TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions +40 29 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges +40 29 TestEventsJournalAccessor +40 29 TestLeaseReclaimerContract +40 29 TestLifecycleCreateContract + +40 30 TestCounterContract +40 30 TestCycleDetectorContract +40 30 TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds +40 30 TestEmbeddedCommitToleratesNothingToCommit +40 30 TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion +40 30 TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState +40 30 TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress +40 30 TestEmbeddedReaderReadySortPoliciesOrderTheSameRows +40 30 TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges +40 30 TestLegacyTrackedIgnoredCursorUnwedgesPull +40 30 TestUpdateMergeOps_DoltEngine_TwoIssues + +40 31 TestDualWriteContract +40 31 TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget +40 31 TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes +40 31 TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn +40 31 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers +40 31 TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing +40 31 TestEmbeddedTransactionWispLifecyclePersistsDurableDependent +40 31 TestEventsSinceEmbedded +40 31 TestGetLabels +40 31 TestReadyCounterContract +40 31 TestWithPeerAuth_KeyMismatchFailsClosed + +40 32 TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone +40 32 TestEmbeddedDependencyEditorRefusesAMissingLocalTarget +40 32 TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers +40 32 TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones +40 32 TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables +40 32 TestEmbeddedReadyClaimerEmptyFrontIsNormal +40 32 TestEmbeddedRecomputeAllBlockedWiring +40 32 TestEventsJournal_EmbeddedConcurrentGaplessNoDup +40 32 TestGetNextChildID +40 32 TestRemoteVerbPlainRemoteKeepsAmbientEnv +40 32 TestWithPeerAuth_MissingRemoteFailsClosed + +40 33 TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex +40 33 TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate +40 33 TestEmbeddedDependencyEditorValidationRefusalsWriteNothing +40 33 TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace +40 33 TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke +40 33 TestEmbeddedRelationsResolvesTheAnchorIDExactly +40 33 TestEmbeddedSyncCommitsConfigBeforeMerge +40 33 TestEventsAutoPruneThrottleSkipsTheSecondPass +40 33 TestGetDependentRecordsEmbedded +40 33 TestPruneEventsBatchDeletesTheLowestSeqsOnly +40 33 TestTreeWalkerContract + +40 34 TestAddLabel +40 34 TestDecryptPassword_KeyMismatchNamesTheLocalKey +40 34 TestEmbeddedBlockedRecheckRecordsUnblockingWrites +40 34 TestEmbeddedDependencyEditorAttributesItsEventsToTheActor +40 34 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp +40 34 TestEmbeddedIssueOperationsCreateAggregatesRelations +40 34 TestEmbeddedReaderReadyDeferredAndEphemeralGates +40 34 TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded +40 34 TestMergeWithStrategy_TheirsResolvesAndCommits +40 34 TestQuerierContract +40 34 TestReadyWorkStatementBudget + +40 35 TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing +40 35 TestEmbeddedClaimIssueWrapperClaimsTheWispPlane +40 35 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType +40 35 TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle +40 35 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse +40 35 TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged +40 35 TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField +40 35 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint +40 35 TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery +40 35 TestReadyWorkWithMissingDeferredParentTables +40 35 TestReclaimExpiredLeaseSurvivesRestartEmbedded + +40 36 TestCommenterContract +40 36 TestConcurrentOpenReturnsCached +40 36 TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory +40 36 TestEmbeddedDependencyEditorRefusalWritesNothing +40 36 TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe +40 36 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext +40 36 TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry +40 36 TestEmbeddedRelationsRefusesAnUnusableTypeFilter +40 36 TestGetReadyWorkMoleculeFilter +40 36 TestReleaserContract +40 36 TestSchemaAfterInit + +40 37 TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch +40 37 TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape +40 37 TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest +40 37 TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop +40 37 TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants +40 37 TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies +40 37 TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance +40 37 TestEmbeddedMergeAndSettleReportsOperatorConflicts +40 37 TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse +40 37 TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch +40 37 TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern + +40 38 TestCreateIssues +40 38 TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt +40 38 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy +40 38 TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose +40 38 TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel +40 38 TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue +40 38 TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge +40 38 TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing +40 38 TestEmbeddedTempRebuildIntermediatesNeverReachHistory +40 38 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff +40 38 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials + +40 39 TestDualWriteContractFlagOff +40 39 TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit +40 39 TestEmbeddedBlockedRecheckFailureKeepsTheWrite +40 39 TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes +40 39 TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone +40 39 TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690 +40 39 TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse +40 39 TestEmbeddedRelationsResolvesAWispAnchor +40 39 TestPreviewStoreRefusesLocalMetadataWrite +40 39 TestReclaimRefusesForeignReplicaLease +40 39 TestWorkspaceConfigContract + +40 40 TestCountsWithMissingWispPlaneTables +40 40 TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest +40 40 TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded +40 40 TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge +40 40 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy +40 40 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane +40 40 TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion +40 40 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers +40 40 TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal +40 40 TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue +40 40 TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 9e24e688b..632b264d0 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1293,11 +1293,12 @@ jobs: # proves nothing pr-risk.yml's embedded jobs (which do run for forks) do # not. # - # Shard counts: storage (20) is the smallest count whose LPT-packed - # heaviest shard (scripts/ci/gen_embedded_storage_shard_manifest.py - # --weights=duration --repack, on rbe-west measurements) stays near 110s, - # under the ~150s per-shard budget with headroom for executor load; its - # heaviest single top-level test is ~90s. cmd is a 100-shard block split + # Shard counts: storage (40): its tests are serial, so a shard's wall + # time is the sum of its tests, and CI shards run ~1.3-1.6x the rbe-west + # measurements; 40 is where the LPT-packed heaviest shard + # (scripts/ci/gen_embedded_storage_shard_manifest.py --weights=duration + # --repack) is its heaviest single top-level test, ~71s + # (TestLifecycleCloseReopenContract). cmd is a 100-shard block split # over two targets, //cmd/bd:bd_embedded_test (shards 1-50) and # //cmd/bd:bd_embedded_part2_test (51-100), because Bazel caps a rule at # 50 shards ("Having more than 50 shards is indicative of poor test @@ -1396,7 +1397,7 @@ jobs: --bep "$RUNNER_TEMP/bazel-bep.json" --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 50@0/100 --suite //cmd/bd:bd_embedded_part2_test .github/scripts/embedded-test-shard.sh 50@50/100 - --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 20 + --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 40 --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index d6e4e1c61..c0c340b6d 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1041,7 +1041,7 @@ Required` requires them to have run remotely and passed. `//cmd/bd:bd_embedded_part2_test` (50 each: shards 1-50 and 51-100 of the manifest's 100-shard block, which their ranges must tile exactly once; the manifest's frozen 20-shard block was the retired - `test-embedded-cmd` jobs' split, F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; + `test-embedded-cmd` jobs' split, F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (40; the frozen 5-shard block was `test-embedded-storage`'s, F1), `//cmd/bd:bd_proxied_test` (34; the frozen 15-shard block was `test-proxied-cmd`'s, F2) and diff --git a/internal/storage/createbatchequiv/createbatchequiv.go b/internal/storage/createbatchequiv/createbatchequiv.go index bbb5c3de8..79f0c1e05 100644 --- a/internal/storage/createbatchequiv/createbatchequiv.go +++ b/internal/storage/createbatchequiv/createbatchequiv.go @@ -100,6 +100,38 @@ func Run(t *testing.T, open Open) { runScenarios(t, open, scenarios()) } +// ScenarioNames returns the names of Run's scenarios, in Run's order. +func ScenarioNames() []string { + var names []string + for _, sc := range scenarios() { + names = append(names, sc.name) + } + return names +} + +// RunNamed is Run for only the named scenarios (in Run's order), for a +// backend that spreads Run's scenarios over several top-level tests; it fails +// on a name Run does not have. Such a backend must cover ScenarioNames() +// exactly once between its tests. +func RunNamed(t *testing.T, open Open, names ...string) { + t.Helper() + want := map[string]bool{} + for _, n := range names { + want[n] = true + } + var list []scenario + for _, sc := range scenarios() { + if want[sc.name] { + list = append(list, sc) + delete(want, sc.name) + } + } + for n := range want { + t.Fatalf("createbatchequiv: no scenario %q (have %v)", n, ScenarioNames()) + } + runScenarios(t, open, list) +} + // RunLarge is Run for the 458-issue scenarios. A backend runs it from its own // top-level test so a race-instrumented lane can skip it (see the embedded // backend's caller) without losing the light scenarios. diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 8241d7414..7997a0f13 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -271,9 +271,9 @@ go_test( ], ) -# Runs 20 Bazel shards against every top-level Test* function in +# Runs 40 Bazel shards against every top-level Test* function in # internal/storage/embeddeddolt/*_test.go but TestConformance, assigned by -# the manifest's 20-shard block (duration-balanced on rbe-west measurements; +# the manifest's 40-shard block (duration-balanced on rbe-west measurements; # see scripts/ci/embedded_storage_test_durations.json), with # BEADS_TEST_EMBEDDED_DOLT=1; the wrapper points # BEADS_TEST_EMBEDDED_TEST_BINARY at this binary, which the concurrency test @@ -310,7 +310,7 @@ sh_test( "//:.github/scripts/embedded-storage-test-shards.txt", ], env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, - shard_count = 20, + shard_count = 40, tags = ["embedded"], ) diff --git a/internal/storage/embeddeddolt/create_batch_equivalence_test.go b/internal/storage/embeddeddolt/create_batch_equivalence_test.go index 913f1ab35..12a6d0214 100644 --- a/internal/storage/embeddeddolt/create_batch_equivalence_test.go +++ b/internal/storage/embeddeddolt/create_batch_equivalence_test.go @@ -4,19 +4,51 @@ package embeddeddolt_test import ( "database/sql" + "slices" "testing" "github.com/steveyegge/beads/internal/storage/createbatchequiv" "github.com/steveyegge/beads/internal/storage/sqlcount" ) -// TestCreateBatchFastPathsMatchPerRow_Embedded runs the batch-create -// equivalence scenario (internal/storage/createbatchequiv) on the embedded -// engine: the same import-shaped batch through the fast and per-row bodies -// must store the same rows. -func TestCreateBatchFastPathsMatchPerRow_Embedded(t *testing.T) { +// createBatchEquivalenceParts spreads createbatchequiv.Run's scenarios over +// the two TestCreateBatchFastPathsMatchPerRow*_Embedded tests below: under +// -race they took ~110s as one test, a whole CI shard on their own. +// TestCreateBatchEquivalencePartsCoverEveryScenario keeps the split exact. +var createBatchEquivalenceParts = [][]string{ + {"small", "depadd"}, + {"apply", "waitsfor"}, +} + +// TestCreateBatchFastPathsMatchPerRowSmallDepAdd_Embedded and +// TestCreateBatchFastPathsMatchPerRowApplyWaitsFor_Embedded run the +// batch-create equivalence scenarios (internal/storage/createbatchequiv) on +// the embedded engine: the same import-shaped batch through the fast and +// per-row bodies must store the same rows. +func TestCreateBatchFastPathsMatchPerRowSmallDepAdd_Embedded(t *testing.T) { skipUnlessEmbeddedDolt(t) - createbatchequiv.Run(t, openEquivalenceDB) + createbatchequiv.RunNamed(t, openEquivalenceDB, createBatchEquivalenceParts[0]...) +} + +func TestCreateBatchFastPathsMatchPerRowApplyWaitsFor_Embedded(t *testing.T) { + skipUnlessEmbeddedDolt(t) + createbatchequiv.RunNamed(t, openEquivalenceDB, createBatchEquivalenceParts[1]...) +} + +// TestCreateBatchEquivalencePartsCoverEveryScenario fails unless the parts +// above name every createbatchequiv.Run scenario exactly once, so a scenario +// added there cannot silently stop running on this backend. +func TestCreateBatchEquivalencePartsCoverEveryScenario(t *testing.T) { + var got []string + for _, part := range createBatchEquivalenceParts { + got = append(got, part...) + } + want := createbatchequiv.ScenarioNames() + slices.Sort(got) + slices.Sort(want) + if !slices.Equal(got, want) { + t.Errorf("createBatchEquivalenceParts cover %v, want each of %v exactly once", got, want) + } } // TestCreateBatchFastPathsMatchPerRowLarge_Embedded runs the 458-issue diff --git a/internal/storage/embeddeddolt/federation_credentials_test.go b/internal/storage/embeddeddolt/federation_credentials_test.go index 27d68cc02..1bbd3300b 100644 --- a/internal/storage/embeddeddolt/federation_credentials_test.go +++ b/internal/storage/embeddeddolt/federation_credentials_test.go @@ -154,7 +154,18 @@ func TestWithPeerAuth(t *testing.T) { } } -func TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword(t *testing.T) { +// The suppressed-ambient-password cases run as two top-level tests, the +// first four cases and the rest: each case opens its own store (~11s under +// -race), so the eight took ~80s as one test. +func TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordStored(t *testing.T) { + testWithPeerAuthWarnsWhenStoredPeerSuppressesAmbientPassword(t, func(i int) bool { return i < 4 }) +} + +func TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordAmbient(t *testing.T) { + testWithPeerAuthWarnsWhenStoredPeerSuppressesAmbientPassword(t, func(i int) bool { return i >= 4 }) +} + +func testWithPeerAuthWarnsWhenStoredPeerSuppressesAmbientPassword(t *testing.T, pick func(i int) bool) { // Pinned verbatim, not matched by substring: the peer name, the username, // the stored tier, and the Warning: prefix are the parts an operator // reads, so a wording change has to be made here deliberately. @@ -252,7 +263,10 @@ func TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword(t *testing.T) }, } - for _, tc := range cases { + for i, tc := range cases { + if !pick(i) { + continue + } t.Run(tc.name, func(t *testing.T) { ctx := t.Context() diff --git a/scripts/ci/embedded_storage_test_durations.json b/scripts/ci/embedded_storage_test_durations.json index 7e3ae6350..631751ebe 100644 --- a/scripts/ci/embedded_storage_test_durations.json +++ b/scripts/ci/embedded_storage_test_durations.json @@ -1,21 +1,17 @@ { "_comment": [ "Measured on rbe-west, the lane's own remote executors (instance oss), on", - "2026-10-09: two `bazel test --config=embedded --nocache_test_results", - "//internal/storage/embeddeddolt:embeddeddolt_embedded_test` runs (race,", - "`-test.parallel=4`), one with the then-committed 15-shard manifest and one", - "with an 18-shard repack, so every one of the 395 top-level Test* functions", - "(excluding TestConformance, sharded separately, and TestMain) ran exactly", - "once per run. Each value is the mean of the two runs; the second ran under", - "heavier executor load and came out ~25% slower across the board, which is", - "the spread to expect between CI runs. Both runs were taken after", - "RunBatchGetterAcceptsExactlyTheCap started seeding its 1000 issues in one", - "CreateIssues call (TestBatchGetterContract: 198.87s -> ~7s on the same", - "executors). The previous capture was local sandboxed execution, and it", - "predated TestBatchGetterContract, so the packer weighted that test at its", - "t.Run fallback (~54s) and stacked a ~200s test onto an already full shard", - "(shard 9 of 15: 352s in CI). Remote shard wall time tracked the packed", - "cost to within 2% in both runs (correlation 0.999+).", + "2026-10-09: one `bazel test //... --config=embedded --nocache_test_results`", + "run (the whole embedded lane at once, as CI runs it: race,", + "`-test.parallel=4`), so every one of the top-level Test* functions", + "(excluding TestConformance, sharded separately) ran exactly once. Nearly", + "every function here is serial (no t.Parallel), so a shard's wall time is", + "the sum of its tests; CI shards ran ~1.3x (up to ~1.6x) these sums.", + "TestCreateBatchFastPathsMatchPerRow_Embedded (111s) and", + "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword (81s) were", + "then split into two top-level tests each; their parts' values are the", + "measured subtest sums (createbatchequiv's scenarios) and half the measured", + "whole (the eight ~11s peer-auth cases split 4+4).", "", "Each value is that top-level test's own from Bazel's", "generated test.xml, NOT the enclosing , for the same", @@ -27,6 +23,8 @@ "capture to without re-verifying that relationship for the", "toolchain in use at the time.", "", + "Notes from earlier captures (their \"this capture\" is the earlier one):", + "", "One originally-measured top-level function,", "TestLargeBatchApplyStatementCounts_Embedded (350.25s: 81.03s + 260.70s +", "8.52s for its 356/712/classic-40 subtests), was subsequently split (same", @@ -84,398 +82,404 @@ "tests": { "TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase": 0.0, "TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase": 0.0, - "TestAddDependency": 24.83, - "TestAddLabel": 0.61, - "TestBatchApplyContract": 62.95, - "TestBatchCreatorContract": 17.7, - "TestBatchGetterContract": 6.37, - "TestBlockingAnnotatorContract": 10.55, - "TestBootstrapperContract": 2.03, + "TestAddDependency": 24.98, + "TestAddLabel": 0.6, + "TestBatchApplyContract": 69.1, + "TestBatchCreatorContract": 6.74, + "TestBatchGetterContract": 6.42, + "TestBlockingAnnotatorContract": 10.85, + "TestBootstrapperContract": 2.24, "TestBuildDSN_NoDatabase": 0.0, "TestBuildDSN_SpacesInPath": 0.0, "TestBuildDSN_WithDatabase": 0.0, "TestCleanGitRemoteCacheGarbage": 0.0, "TestCleanGitRemoteCacheGarbage_NoCacheDir": 0.0, "TestCleanGitRemoteCacheGarbage_Throttled": 0.0, - "TestCommenterContract": 4.51, - "TestCommitAllOutOfBandChangeAndHonestNoOp": 0.58, - "TestConcurrencyMultiProcess": 4.32, - "TestConcurrentOpenReturnsCached": 11.14, - "TestCounterContract": 21.42, - "TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra": 0.8, - "TestCountsIncludeWispDependencies": 1.12, - "TestCountsWithMissingWispPlaneTables": 1.93, + "TestCommenterContract": 4.86, + "TestCommitAllOutOfBandChangeAndHonestNoOp": 0.67, + "TestConcurrencyMultiProcess": 3.67, + "TestConcurrentOpenReturnsCached": 11.52, + "TestCounterContract": 17.83, + "TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra": 0.87, + "TestCountsIncludeWispDependencies": 1.06, + "TestCountsWithMissingWispPlaneTables": 2.65, + "TestCreateBatchEquivalencePartsCoverEveryScenario": 0.01, + "TestCreateBatchFastPathsMatchPerRowApplyWaitsFor_Embedded": 51.46, "TestCreateBatchFastPathsMatchPerRowLarge_Embedded": 0.0, - "TestCreateBatchFastPathsMatchPerRow_Embedded": 89.57, - "TestCreateIssue": 21.51, - "TestCreateIssues": 16.54, - "TestCreateIssuesRejectStaleUpserts": 3.85, - "TestCreateRejectsCrossTableIDCollision": 1.54, + "TestCreateBatchFastPathsMatchPerRowSmallDepAdd_Embedded": 59.56, + "TestCreateIssue": 23.52, + "TestCreateIssues": 9.65, + "TestCreateIssuesRejectStaleUpserts": 4.25, + "TestCreateRejectsCrossTableIDCollision": 1.6, "TestCrossRecordInvariantContract": 0.0, - "TestCycleDetectorContract": 7.24, - "TestDecryptPassword_KeyMismatchNamesTheLocalKey": 10.81, - "TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch": 11.12, - "TestDeleterContract": 42.31, + "TestCycleDetectorContract": 7.58, + "TestDecryptPassword_KeyMismatchNamesTheLocalKey": 11.45, + "TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch": 11.11, + "TestDeleterContract": 45.61, "TestDepRekeyMarkerStillInIgnoredSeries": 0.0, - "TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles": 1.83, + "TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles": 3.02, "TestDoltCommitResponseLossIsIndeterminate": 0.0, "TestDualWriteContract": 1.46, "TestDualWriteContractFlagOff": 0.46, - "TestDualWriteFixtureKitIsWired": 0.21, - "TestEdgeReaderContract": 6.2, - "TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing": 12.17, - "TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory": 0.89, - "TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes": 10.96, - "TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure": 11.11, - "TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed": 1.48, - "TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed": 0.52, - "TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty": 6.01, - "TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch": 13.75, - "TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest": 5.79, - "TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex": 12.23, - "TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp": 6.46, - "TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy": 9.44, - "TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess": 3.06, - "TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape": 12.34, - "TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex": 12.61, - "TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit": 13.57, - "TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded": 6.56, - "TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing": 11.7, - "TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed": 13.28, - "TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim": 1.27, - "TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges": 1.24, - "TestEmbeddedBlockedRecheckFailureKeepsTheWrite": 11.59, - "TestEmbeddedBlockedRecheckRecordsUnblockingWrites": 14.75, - "TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded": 12.69, - "TestEmbeddedCheckedWrappersRouteTheirPreconditions": 3.17, - "TestEmbeddedClaimIssueWrapperClaimsTheWispPlane": 0.79, - "TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne": 1.13, - "TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow": 0.8, - "TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone": 1.36, - "TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting": 0.72, - "TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting": 0.92, - "TestEmbeddedClaimerRefusalMessagesCarryTheirFragments": 1.02, - "TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt": 0.82, - "TestEmbeddedClaimerRefusesAWispIDAsNotFound": 0.58, - "TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState": 0.64, - "TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween": 1.11, - "TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder": 1.42, - "TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds": 0.69, - "TestEmbeddedCommitToleratesNothingToCommit": 0.53, - "TestEmbeddedConfigCommitStagesConfig": 0.33, - "TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded": 8.79, - "TestEmbeddedDepRekeyConvergesRenameChain": 8.79, - "TestEmbeddedDepRekeyMergesFreshDuplicateEdges": 15.36, - "TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies": 7.66, - "TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge": 8.19, - "TestEmbeddedDependencyEditorAcceptsADiamond": 3.03, - "TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget": 0.6, - "TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants": 1.26, - "TestEmbeddedDependencyEditorAcceptsAnExternalTarget": 0.59, - "TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes": 2.3, - "TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb": 1.9, - "TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder": 1.85, - "TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges": 2.17, - "TestEmbeddedDependencyEditorAttributesItsEventsToTheActor": 1.82, - "TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate": 4.26, - "TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType": 1.91, - "TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes": 4.53, - "TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes": 1.35, - "TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes": 1.11, - "TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest": 0.69, - "TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest": 1.0, - "TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest": 2.4, - "TestEmbeddedDependencyEditorRefusalWritesNothing": 1.38, - "TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop": 5.9, - "TestEmbeddedDependencyEditorRefusesAGhostSource": 0.66, - "TestEmbeddedDependencyEditorRefusesAMissingLocalTarget": 1.15, - "TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle": 2.52, - "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy": 2.06, - "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy": 1.78, - "TestEmbeddedDependencyEditorRefusesCrossPlaneCycle": 0.98, - "TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped": 0.49, - "TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked": 2.53, - "TestEmbeddedDependencyEditorRemoveIsIdempotent": 2.13, - "TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants": 4.83, - "TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge": 1.19, - "TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse": 1.57, - "TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge": 0.92, - "TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane": 0.78, - "TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane": 1.61, - "TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent": 1.23, - "TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion": 1.36, - "TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion": 1.6, - "TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp": 2.17, - "TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe": 0.98, - "TestEmbeddedDependencyEditorSnapshotsTheRequest": 0.73, - "TestEmbeddedDependencyEditorValidationRefusalsWriteNothing": 0.72, - "TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove": 2.29, - "TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn": 1.89, - "TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers": 15.34, + "TestDualWriteFixtureKitIsWired": 0.27, + "TestEdgeReaderContract": 6.25, + "TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing": 11.52, + "TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory": 0.94, + "TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes": 10.75, + "TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure": 10.96, + "TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed": 12.01, + "TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed": 10.82, + "TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty": 0.97, + "TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch": 12.85, + "TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest": 11.58, + "TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex": 11.43, + "TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp": 12.21, + "TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy": 3.76, + "TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess": 3.04, + "TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards": 3.1, + "TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape": 11.59, + "TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex": 12.02, + "TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit": 13.52, + "TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded": 1.54, + "TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing": 10.84, + "TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed": 7.47, + "TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim": 11.6, + "TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges": 1.58, + "TestEmbeddedBlockedRecheckFailureKeepsTheWrite": 13.14, + "TestEmbeddedBlockedRecheckRecordsUnblockingWrites": 12.78, + "TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded": 13.15, + "TestEmbeddedCheckedWrappersRouteTheirPreconditions": 3.2, + "TestEmbeddedClaimIssueWrapperClaimsTheWispPlane": 12.81, + "TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne": 1.2, + "TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow": 10.87, + "TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone": 1.41, + "TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting": 0.74, + "TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting": 0.95, + "TestEmbeddedClaimerRefusalMessagesCarryTheirFragments": 1.08, + "TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt": 0.94, + "TestEmbeddedClaimerRefusesAWispIDAsNotFound": 0.62, + "TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState": 0.67, + "TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween": 1.52, + "TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder": 19.05, + "TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds": 12.47, + "TestEmbeddedCommitToleratesNothingToCommit": 0.64, + "TestEmbeddedConfigCommitStagesConfig": 0.32, + "TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded": 10.62, + "TestEmbeddedDepRekeyConvergesRenameChain": 9.83, + "TestEmbeddedDepRekeyMergesFreshDuplicateEdges": 18.83, + "TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies": 11.97, + "TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge": 9.6, + "TestEmbeddedDependencyEditorAcceptsADiamond": 2.32, + "TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget": 0.67, + "TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants": 0.99, + "TestEmbeddedDependencyEditorAcceptsAnExternalTarget": 0.67, + "TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes": 2.22, + "TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb": 1.89, + "TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder": 2.05, + "TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges": 1.98, + "TestEmbeddedDependencyEditorAttributesItsEventsToTheActor": 1.9, + "TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate": 3.93, + "TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType": 1.98, + "TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes": 4.65, + "TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes": 1.16, + "TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes": 1.13, + "TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest": 0.99, + "TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest": 1.13, + "TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest": 2.47, + "TestEmbeddedDependencyEditorRefusalWritesNothing": 1.43, + "TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop": 9.78, + "TestEmbeddedDependencyEditorRefusesAGhostSource": 0.87, + "TestEmbeddedDependencyEditorRefusesAMissingLocalTarget": 0.88, + "TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle": 2.95, + "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy": 1.98, + "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy": 2.01, + "TestEmbeddedDependencyEditorRefusesCrossPlaneCycle": 1.05, + "TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped": 0.54, + "TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked": 2.3, + "TestEmbeddedDependencyEditorRemoveIsIdempotent": 2.31, + "TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants": 5.28, + "TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge": 1.11, + "TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse": 1.45, + "TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge": 0.98, + "TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane": 0.82, + "TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane": 2.13, + "TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent": 1.36, + "TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion": 1.4, + "TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion": 1.59, + "TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp": 2.39, + "TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe": 1.16, + "TestEmbeddedDependencyEditorSnapshotsTheRequest": 0.78, + "TestEmbeddedDependencyEditorValidationRefusalsWriteNothing": 0.61, + "TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove": 2.4, + "TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn": 1.96, + "TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers": 16.29, "TestEmbeddedDoltStoreIssueLifecycleBuildsOperations": 0.0, "TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore": 0.0, - "TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion": 3.83, - "TestEmbeddedGetMergeBlockersOnAQuietStore": 0.3, - "TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches": 2.06, - "TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones": 8.68, - "TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace": 9.18, - "TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables": 6.47, - "TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape": 7.57, - "TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape": 29.07, - "TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone": 2.19, - "TestEmbeddedIssueOperationsCloseAndReopenAWisp": 1.33, - "TestEmbeddedIssueOperationsCreateAggregatesRelations": 1.42, - "TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp": 1.34, - "TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies": 1.56, - "TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure": 2.22, - "TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables": 1.16, - "TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps": 1.73, - "TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID": 3.79, - "TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction": 3.06, - "TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults": 3.54, - "TestEmbeddedIssueOperationsRequestValuesAreNotMutated": 2.19, - "TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged": 9.04, - "TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix": 2.15, - "TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState": 3.17, - "TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses": 2.12, - "TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim": 0.65, - "TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose": 9.43, - "TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent": 1.36, - "TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel": 0.52, - "TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps": 0.98, - "TestEmbeddedIssueOperationsUpdateLabelPatchOrdering": 1.06, - "TestEmbeddedIssueOperationsUpdateLabelPatchValueRules": 1.24, - "TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates": 1.93, - "TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel": 0.65, - "TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure": 6.98, - "TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes": 0.77, - "TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary": 1.62, - "TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress": 1.03, - "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender": 5.16, - "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers": 3.93, - "TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField": 1.6, - "TestEmbeddedLenientOpenWarning_DataBehind": 55.67, - "TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance": 10.91, - "TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge": 0.45, - "TestEmbeddedMergeAndSettleFKCascadeRepair": 1.89, - "TestEmbeddedMergeAndSettleMemoryConfigConflict": 0.47, - "TestEmbeddedMergeAndSettleMetadataConflict": 0.45, - "TestEmbeddedMergeAndSettleRefusesUnknownTable": 0.8, - "TestEmbeddedMergeAndSettleReportsOperatorConflicts": 0.69, - "TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict": 0.49, - "TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers": 14.45, - "TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566": 18.94, - "TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690": 13.55, - "TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566": 7.62, - "TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566": 8.24, - "TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain": 10.21, - "TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext": 10.7, - "TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation": 6.17, - "TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints": 7.11, - "TestEmbeddedMigration0058HealsAlreadyAffectedDatabase": 6.63, - "TestEmbeddedMigration0062FlipsEventsToIgnoredPlane": 5.08, - "TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke": 11.27, - "TestEmbeddedOpenForReadOnlyCommand_LenientGate": 10.66, - "TestEmbeddedOpenForRemoteSync_DataBehind": 21.05, - "TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal": 10.86, + "TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion": 11.08, + "TestEmbeddedGetMergeBlockersOnAQuietStore": 0.33, + "TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches": 1.76, + "TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones": 9.22, + "TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace": 9.27, + "TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables": 8.49, + "TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape": 9.36, + "TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape": 21.5, + "TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone": 2.74, + "TestEmbeddedIssueOperationsCloseAndReopenAWisp": 1.32, + "TestEmbeddedIssueOperationsCreateAggregatesRelations": 1.4, + "TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp": 1.05, + "TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies": 1.95, + "TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure": 2.61, + "TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables": 1.18, + "TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps": 2.08, + "TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID": 4.06, + "TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction": 3.48, + "TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults": 3.99, + "TestEmbeddedIssueOperationsRequestValuesAreNotMutated": 1.75, + "TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged": 9.43, + "TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix": 1.85, + "TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState": 2.5, + "TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses": 1.71, + "TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim": 0.62, + "TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose": 11.59, + "TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent": 1.49, + "TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel": 0.51, + "TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps": 1.16, + "TestEmbeddedIssueOperationsUpdateLabelPatchOrdering": 1.04, + "TestEmbeddedIssueOperationsUpdateLabelPatchValueRules": 2.4, + "TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates": 1.75, + "TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel": 0.49, + "TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure": 6.15, + "TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes": 0.94, + "TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary": 1.17, + "TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress": 1.1, + "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender": 5.4, + "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers": 5.3, + "TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField": 1.16, + "TestEmbeddedLenientOpenWarning_DataBehind": 49.93, + "TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance": 13.58, + "TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge": 0.53, + "TestEmbeddedMergeAndSettleFKCascadeRepair": 1.71, + "TestEmbeddedMergeAndSettleMemoryConfigConflict": 0.49, + "TestEmbeddedMergeAndSettleMetadataConflict": 0.43, + "TestEmbeddedMergeAndSettleRefusesUnknownTable": 0.5, + "TestEmbeddedMergeAndSettleReportsOperatorConflicts": 0.86, + "TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict": 0.54, + "TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers": 16.75, + "TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566": 22.49, + "TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690": 9.59, + "TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566": 10.78, + "TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566": 10.49, + "TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain": 10.53, + "TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext": 14.21, + "TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation": 6.55, + "TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints": 6.64, + "TestEmbeddedMigration0058HealsAlreadyAffectedDatabase": 6.74, + "TestEmbeddedMigration0062FlipsEventsToIgnoredPlane": 5.98, + "TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke": 12.76, + "TestEmbeddedOpenForReadOnlyCommand_LenientGate": 20.67, + "TestEmbeddedOpenForRemoteSync_DataBehind": 31.77, + "TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal": 13.44, "TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites": 0.0, - "TestEmbeddedOpenReadOnly_SkipsGateAndMigrations": 12.74, - "TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape": 36.7, - "TestEmbeddedReaderDoesNotMutateTheCallerRequest": 0.57, - "TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows": 1.08, - "TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue": 1.0, - "TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay": 0.32, - "TestEmbeddedReaderGetOptionalRowListsAreOffByDefault": 1.44, - "TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes": 1.46, - "TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse": 1.95, - "TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge": 3.36, - "TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides": 3.68, - "TestEmbeddedReaderListEmptyPageIsWellFormed": 0.46, - "TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder": 3.81, - "TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates": 2.31, - "TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder": 0.92, - "TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing": 3.0, - "TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress": 1.82, - "TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset": 2.54, - "TestEmbeddedReaderListMaxRowsIsHonored": 1.65, - "TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch": 2.14, - "TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn": 1.26, - "TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing": 2.44, - "TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet": 1.31, - "TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters": 1.83, - "TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry": 1.79, - "TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary": 0.7, - "TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse": 2.25, - "TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet": 1.23, - "TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt": 1.58, - "TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage": 2.27, - "TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse": 1.09, - "TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType": 0.95, - "TestEmbeddedReaderReadyDeferredAndEphemeralGates": 1.2, - "TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary": 2.49, - "TestEmbeddedReaderReadyLimitBoundary": 1.18, - "TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded": 4.48, - "TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix": 44.08, - "TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants": 3.71, - "TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions": 1.77, - "TestEmbeddedReaderReadySortPoliciesOrderTheSameRows": 0.53, - "TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists": 1.05, - "TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits": 0.95, - "TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState": 1.57, - "TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest": 0.34, - "TestEmbeddedReadyClaimerEmptyFrontIsNormal": 0.72, - "TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven": 3.13, - "TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities": 3.83, - "TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne": 1.87, - "TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet": 0.73, - "TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin": 0.78, - "TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor": 0.46, - "TestEmbeddedReadyClaimerSkipsIneligibleFrontRows": 1.04, - "TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped": 5.52, - "TestEmbeddedRecomputeAllBlockedWiring": 1.79, - "TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection": 1.54, - "TestEmbeddedRelationsAnswersInThePinnedOrder": 1.21, - "TestEmbeddedRelationsDirectionSelectsTheInverseGraph": 1.0, - "TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary": 0.8, - "TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer": 1.16, - "TestEmbeddedRelationsLeavesTheCallersRequestAlone": 0.81, - "TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether": 1.47, - "TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength": 0.4, - "TestEmbeddedRelationsRefusesAnUnusableTypeFilter": 0.42, - "TestEmbeddedRelationsRefusesTheZeroDirection": 0.46, - "TestEmbeddedRelationsResolvesAWispAnchor": 1.03, - "TestEmbeddedRelationsResolvesTheAnchorIDExactly": 1.09, - "TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue": 0.42, - "TestEmbeddedRemoteMigrateGate_BlocksReopen": 9.98, - "TestEmbeddedRenameKeepsDependencyIDsDeterministic": 8.71, - "TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing": 0.94, - "TestEmbeddedReopenIssueCategorySemantics": 3.54, - "TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit": 0.62, - "TestEmbeddedRoleFixtureKitHooksAreUsable": 1.35, + "TestEmbeddedOpenReadOnly_SkipsGateAndMigrations": 11.86, + "TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape": 37.92, + "TestEmbeddedReaderDoesNotMutateTheCallerRequest": 0.67, + "TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows": 1.25, + "TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue": 1.42, + "TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay": 0.31, + "TestEmbeddedReaderGetOptionalRowListsAreOffByDefault": 1.39, + "TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes": 1.1, + "TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse": 2.65, + "TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge": 4.98, + "TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides": 4.16, + "TestEmbeddedReaderListEmptyPageIsWellFormed": 0.51, + "TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder": 5.75, + "TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates": 3.44, + "TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder": 1.02, + "TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing": 2.88, + "TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress": 1.54, + "TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset": 2.55, + "TestEmbeddedReaderListMaxRowsIsHonored": 1.64, + "TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch": 1.48, + "TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn": 1.59, + "TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing": 2.87, + "TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet": 1.4, + "TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters": 1.73, + "TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry": 2.68, + "TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary": 0.85, + "TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse": 2.02, + "TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet": 2.1, + "TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt": 1.72, + "TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage": 2.48, + "TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse": 1.06, + "TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType": 1.54, + "TestEmbeddedReaderReadyDeferredAndEphemeralGates": 1.19, + "TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary": 3.91, + "TestEmbeddedReaderReadyLimitBoundary": 1.22, + "TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded": 4.24, + "TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix": 43.78, + "TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants": 3.65, + "TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions": 1.79, + "TestEmbeddedReaderReadySortPoliciesOrderTheSameRows": 0.84, + "TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists": 1.37, + "TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits": 1.02, + "TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState": 1.84, + "TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest": 0.39, + "TestEmbeddedReadyClaimerEmptyFrontIsNormal": 0.6, + "TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven": 3.36, + "TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities": 3.65, + "TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne": 2.06, + "TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet": 1.04, + "TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin": 0.93, + "TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor": 0.72, + "TestEmbeddedReadyClaimerSkipsIneligibleFrontRows": 1.36, + "TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped": 10.83, + "TestEmbeddedRecomputeAllBlockedWiring": 1.91, + "TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection": 1.32, + "TestEmbeddedRelationsAnswersInThePinnedOrder": 1.31, + "TestEmbeddedRelationsDirectionSelectsTheInverseGraph": 1.12, + "TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary": 1.58, + "TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer": 1.28, + "TestEmbeddedRelationsLeavesTheCallersRequestAlone": 1.12, + "TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether": 1.66, + "TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength": 0.67, + "TestEmbeddedRelationsRefusesAnUnusableTypeFilter": 0.5, + "TestEmbeddedRelationsRefusesTheZeroDirection": 0.75, + "TestEmbeddedRelationsResolvesAWispAnchor": 0.96, + "TestEmbeddedRelationsResolvesTheAnchorIDExactly": 1.2, + "TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue": 0.67, + "TestEmbeddedRemoteMigrateGate_BlocksReopen": 11.87, + "TestEmbeddedRenameKeepsDependencyIDsDeterministic": 9.9, + "TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing": 1.2, + "TestEmbeddedReopenIssueCategorySemantics": 6.31, + "TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit": 0.53, + "TestEmbeddedRoleFixtureKitHooksAreUsable": 1.32, "TestEmbeddedSQLCommitResponseLossIsIndeterminate": 0.0, - "TestEmbeddedSyncCommitsConfigBeforeMerge": 0.7, - "TestEmbeddedTempRebuildIntermediatesNeverReachHistory": 10.07, - "TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges": 0.86, - "TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit": 2.33, - "TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges": 1.52, - "TestEmbeddedTransactionWispLifecyclePersistsDurableDependent": 6.16, + "TestEmbeddedSyncCommitsConfigBeforeMerge": 0.87, + "TestEmbeddedTempRebuildIntermediatesNeverReachHistory": 13.03, + "TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges": 1.06, + "TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit": 2.22, + "TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges": 1.79, + "TestEmbeddedTransactionWispLifecyclePersistsDurableDependent": 12.61, "TestEpochContract": 0.0, - "TestEventsAutoPruneBoundsTheJournalToTheFloors": 1.73, - "TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded": 0.97, - "TestEventsAutoPruneThrottleSkipsTheSecondPass": 1.57, - "TestEventsJournalAccessor": 1.66, - "TestEventsJournalPruneNeverSplitsASkewedPair": 1.48, - "TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff": 0.97, - "TestEventsJournalReadRefusesAnInteriorHole": 1.14, - "TestEventsJournalRestartAndRetentionBoundary": 1.46, - "TestEventsJournal_ActorMatrixOnEmbedded": 4.21, - "TestEventsJournal_EmbeddedConcurrentGaplessNoDup": 2.6, - "TestEventsSinceEmbedded": 0.98, + "TestEventsAutoPruneBoundsTheJournalToTheFloors": 1.68, + "TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded": 1.02, + "TestEventsAutoPruneThrottleSkipsTheSecondPass": 2.34, + "TestEventsJournalAccessor": 1.59, + "TestEventsJournalPruneNeverSplitsASkewedPair": 1.67, + "TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff": 1.17, + "TestEventsJournalReadRefusesAnInteriorHole": 1.2, + "TestEventsJournalRestartAndRetentionBoundary": 1.68, + "TestEventsJournal_ActorMatrixOnEmbedded": 4.08, + "TestEventsJournal_EmbeddedConcurrentGaplessNoDup": 2.33, + "TestEventsSinceEmbedded": 0.79, "TestExpectedRevisionContract": 0.0, - "TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree": 24.77, - "TestForeignSkipDetailIsBounded": 5.58, - "TestGetDependentRecordsEmbedded": 2.15, - "TestGetDependentRecordsForIssuesEmbedded": 3.4, - "TestGetIssue": 2.37, - "TestGetIssueCommentsPageEmbedded": 3.1, - "TestGetLabels": 1.25, - "TestGetNextChildID": 3.75, - "TestGetReadyWorkIncludeEphemeralAppliesWispFilters": 6.5, - "TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps": 1.58, - "TestGetReadyWorkMoleculeFilter": 1.61, + "TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree": 25.13, + "TestForeignSkipDetailIsBounded": 6.11, + "TestGetDependentRecordsEmbedded": 1.95, + "TestGetDependentRecordsForIssuesEmbedded": 3.62, + "TestGetIssue": 2.57, + "TestGetIssueCommentsPageEmbedded": 3.22, + "TestGetLabels": 1.16, + "TestGetNextChildID": 3.83, + "TestGetReadyWorkIncludeEphemeralAppliesWispFilters": 5.96, + "TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps": 2.25, + "TestGetReadyWorkMoleculeFilter": 2.03, "TestGitRefPinsMatchDolt": 0.0, - "TestGraphCounterContract": 9.86, - "TestGraphWalkPlansHonorJoinHints": 5.53, + "TestGraphCounterContract": 9.1, + "TestGraphWalkPlansHonorJoinHints": 5.34, "TestHasRepository": 0.0, - "TestHeartbeatMatchesActorAcrossSpellingEmbedded": 0.56, - "TestHeartbeatRejectsWispEmbedded": 0.29, + "TestHeartbeatMatchesActorAcrossSpellingEmbedded": 0.63, + "TestHeartbeatRejectsWispEmbedded": 0.55, "TestHelperProcess": 0.0, - "TestHistoryPresenceContract": 1.96, - "TestHistory_NullTextColumns": 0.97, - "TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore": 1.86, - "TestIgnoredCursorHealConvergesFromEveryCrashState": 3.68, - "TestIgnoredCursorHealIsIdempotent": 1.62, - "TestIgnoredCursorHealNeverResurrectsRolledBackVersions": 0.92, - "TestIgnoredCursorHealRespectsOperatorOverride": 0.55, - "TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern": 0.44, - "TestIgnoredCursorHealResumeRespectsOperatorOverride": 0.7, - "TestIsBlockedBatchEmbedded": 2.12, - "TestJournalContract": 9.43, - "TestLargeBatchApplyStatementCounts356_Embedded": 22.91, - "TestLargeBatchApplyStatementCounts712_Embedded": 1.0, - "TestLargeBatchApplyStatementCountsClassic40_Embedded": 5.79, - "TestLargeBatchApplyWallClock_Embedded": 21.85, + "TestHistoryPresenceContract": 1.94, + "TestHistory_NullTextColumns": 0.98, + "TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore": 2.3, + "TestIgnoredCursorHealConvergesFromEveryCrashState": 3.34, + "TestIgnoredCursorHealIsIdempotent": 1.35, + "TestIgnoredCursorHealNeverResurrectsRolledBackVersions": 1.21, + "TestIgnoredCursorHealRespectsOperatorOverride": 0.66, + "TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern": 0.63, + "TestIgnoredCursorHealResumeRespectsOperatorOverride": 1.05, + "TestIsBlockedBatchEmbedded": 2.23, + "TestJournalContract": 6.39, + "TestLargeBatchApplyStatementCounts356_Embedded": 23.03, + "TestLargeBatchApplyStatementCounts712_Embedded": 0.0, + "TestLargeBatchApplyStatementCountsClassic40_Embedded": 6.4, + "TestLargeBatchApplyWallClock_Embedded": 23.45, "TestLargeGraphCreateTiming_Embedded": 0.0, - "TestLeaseLifecycleEmbedded": 3.01, - "TestLegacyTrackedIgnoredCursorHealsOnOpen": 0.83, - "TestLegacyTrackedIgnoredCursorUnwedgesPull": 4.25, + "TestLeaseLifecycleEmbedded": 3.29, + "TestLeaseReclaimerContract": 12.41, + "TestLegacyTrackedIgnoredCursorHealsOnOpen": 0.98, + "TestLegacyTrackedIgnoredCursorUnwedgesPull": 4.39, "TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance": 0.0, "TestLenientGateWarningBody_ShapedDecisions": 0.0, "TestLenientGateWarning_ComposedOutput": 0.0, - "TestLifecycleCloseReopenContract": 64.3, - "TestLifecycleCreateContract": 9.5, - "TestLifecycleUpdateContract": 32.78, - "TestListWisps": 1.0, - "TestMemoriesContract": 10.65, - "TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle": 1.06, - "TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict": 1.14, - "TestMergeWithStrategy_OursKeepsLocal": 0.65, - "TestMergeWithStrategy_TheirsResolvesAndCommits": 0.61, - "TestMerge_NoStrategyStillErrorsWithEscapeHatchHint": 0.52, - "TestMetadataCASContract": 9.64, - "TestOpenAfterCloseCreatesNewStore": 11.35, - "TestOpenDifferentDirsReturnsDifferentStores": 22.39, - "TestOpenReturnsCachedStore": 10.72, - "TestPartitionWispIDs": 0.75, - "TestPreviewStoreRefusesLocalMetadataWrite": 11.12, - "TestPristineEmbeddedDoltFixtureRelocatesPrefixes": 1.63, - "TestPromoteClearsNoHistory": 1.77, - "TestPromoteStillWorksWithCollisionGuard": 6.08, - "TestPruneEventsBatchDeletesTheLowestSeqsOnly": 1.23, - "TestPushCommitsPendingChangesFirst": 59.03, - "TestQuerierContract": 7.95, - "TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery": 0.63, - "TestReadyCounterContract": 4.93, - "TestReadyWorkPageTotal": 21.9, - "TestReadyWorkPageTotalFilterMatrix": 15.52, - "TestReadyWorkStatementBudget": 14.59, - "TestReadyWorkWithMissingDeferredParentTables": 4.54, - "TestReclaimAnyReplicaOverride": 0.71, - "TestReclaimExpiredLeaseSurvivesRestartEmbedded": 13.82, - "TestReclaimRefusesForeignReplicaLease": 0.74, - "TestReleaserContract": 5.0, + "TestLifecycleCloseReopenContract": 71.34, + "TestLifecycleCreateContract": 7.0, + "TestLifecycleUpdateContract": 39.96, + "TestListWisps": 1.01, + "TestMain": 0.0, + "TestMemoriesContract": 8.73, + "TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle": 0.73, + "TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict": 0.8, + "TestMergeWithStrategy_OursKeepsLocal": 0.8, + "TestMergeWithStrategy_TheirsResolvesAndCommits": 0.87, + "TestMerge_NoStrategyStillErrorsWithEscapeHatchHint": 0.66, + "TestMetadataCASContract": 10.0, + "TestOpenAfterCloseCreatesNewStore": 10.55, + "TestOpenDifferentDirsReturnsDifferentStores": 24.94, + "TestOpenReturnsCachedStore": 11.77, + "TestPartitionWispIDs": 0.81, + "TestPreviewStoreRefusesLocalMetadataWrite": 11.56, + "TestPristineEmbeddedDoltFixtureRelocatesPrefixes": 1.75, + "TestPromoteClearsNoHistory": 2.06, + "TestPromoteStillWorksWithCollisionGuard": 1.02, + "TestPruneEventsBatchDeletesTheLowestSeqsOnly": 1.37, + "TestPushCommitsPendingChangesFirst": 59.73, + "TestQuerierContract": 9.42, + "TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery": 0.83, + "TestReadyCounterContract": 7.64, + "TestReadyWorkPageTotal": 21.37, + "TestReadyWorkPageTotalFilterMatrix": 29.46, + "TestReadyWorkStatementBudget": 15.52, + "TestReadyWorkWithMissingDeferredParentTables": 4.42, + "TestReclaimAnyReplicaOverride": 0.63, + "TestReclaimExpiredLeaseSurvivesRestartEmbedded": 14.58, + "TestReclaimRefusesForeignReplicaLease": 1.42, + "TestReleaserContract": 9.54, "TestRemoteEntryPointsUseVersionControlOps": 0.0, - "TestRemoteGitDataRefOnEngine": 1.4, - "TestRemoteVerbPlainRemoteKeepsAmbientEnv": 10.97, - "TestRemoteVerbPlainRemoteWithoutAuthPresentsNone": 12.14, - "TestRemoteVerbsPresentStoredPeerCredentials": 12.78, - "TestRenameLabel": 1.98, - "TestRenameLabelMintsOneVersionPerTouchedBead": 2.45, - "TestReopenIssue": 1.09, + "TestRemoteGitDataRefOnEngine": 3.68, + "TestRemoteVerbPlainRemoteKeepsAmbientEnv": 12.73, + "TestRemoteVerbPlainRemoteWithoutAuthPresentsNone": 11.77, + "TestRemoteVerbsPresentStoredPeerCredentials": 23.55, + "TestRenameLabel": 1.91, + "TestRenameLabelMintsOneVersionPerTouchedBead": 2.08, + "TestReopenIssue": 1.36, "TestRetentionContract": 0.0, - "TestRunInTransactionPostCommitStageFailureIsIndeterminate": 0.44, - "TestSchemaAfterInit": 10.4, - "TestSearchWithMissingWispPlaneTables": 1.92, + "TestRunInTransactionPostCommitStageFailureIsIndeterminate": 0.82, + "TestSchemaAfterInit": 12.85, + "TestSearchWithMissingWispPlaneTables": 2.23, "TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed": 0.0, - "TestStatsReporterContract": 9.56, - "TestSweeperContract": 17.21, + "TestStatsReporterContract": 10.44, + "TestSweeperContract": 20.03, "TestTransactionCleanupAfterSQLCommitIsIndeterminate": 0.0, "TestTransactionCleanupBeforeSQLCommitRemainsDefinite": 0.0, - "TestTreeWalkerContract": 12.82, - "TestUnknownProvenanceLeaseStaysReclaimable": 1.21, - "TestUpdateIssueType": 0.51, - "TestUpdateMergeOps_DoltEngine": 0.49, - "TestUpdateMergeOps_DoltEngine_TwoIssues": 0.77, - "TestVersionReconcilerContract": 2.4, - "TestWithPeerAuth": 51.7, - "TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials": 12.63, - "TestWithPeerAuth_DivergedRemoteURLFailsClosed": 12.79, - "TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials": 8.72, - "TestWithPeerAuth_KeyMismatchFailsClosed": 11.39, - "TestWithPeerAuth_MissingRemoteFailsClosed": 10.01, - "TestWithPeerAuth_RestoresAbsentEnv": 11.38, - "TestWithPeerAuth_RestoresEnvWhenCallbackFails": 12.11, - "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword": 61.05, - "TestWorkspaceConfigContract": 5.37 + "TestTreeWalkerContract": 16.04, + "TestUnknownProvenanceLeaseStaysReclaimable": 1.03, + "TestUpdateIssueType": 0.49, + "TestUpdateMergeOps_DoltEngine": 0.59, + "TestUpdateMergeOps_DoltEngine_TwoIssues": 1.56, + "TestVersionReconcilerContract": 3.07, + "TestWithPeerAuth": 58.71, + "TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials": 13.55, + "TestWithPeerAuth_DivergedRemoteURLFailsClosed": 11.6, + "TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials": 12.18, + "TestWithPeerAuth_KeyMismatchFailsClosed": 11.21, + "TestWithPeerAuth_MissingRemoteFailsClosed": 11.39, + "TestWithPeerAuth_RestoresAbsentEnv": 12.08, + "TestWithPeerAuth_RestoresEnvWhenCallbackFails": 12.25, + "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordAmbient": 40.51, + "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordStored": 40.51, + "TestWorkspaceConfigContract": 5.12 } } diff --git a/scripts/ci/gen_embedded_storage_shard_manifest.py b/scripts/ci/gen_embedded_storage_shard_manifest.py index 3df3eba97..a90834356 100755 --- a/scripts/ci/gen_embedded_storage_shard_manifest.py +++ b/scripts/ci/gen_embedded_storage_shard_manifest.py @@ -168,7 +168,7 @@ def main(): ap.error(f'total_shards is required with --write (the default, {default_total_shards}, ' 'is the FROZEN legacy block -- see its header in embedded-storage-test-shards.txt; ' 'it must not be regenerated). Pass the Bazel-only total explicitly instead, e.g. ' - '"15 --weights=duration" for bazel-embedded\'s block (see ' + '"40 --weights=duration" for bazel-embedded\'s block (see ' 'internal/storage/embeddeddolt/BUILD.bazel\'s embeddeddolt_embedded_test ' 'shard_count for the current value)') args.total_shards = default_total_shards diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go index 134fbf905..dd6771c67 100644 --- a/scripts/shard_manifest_test.go +++ b/scripts/shard_manifest_test.go @@ -19,7 +19,7 @@ import ( // S3 (F1, mirroring F2's TestProxiedShardManifestGeneratorNotStale below — // see that test's doc comment for the full --check rationale, not repeated -// here): the Bazel-only 50-shard cmd block and 20-shard storage block are +// here): the Bazel-only 100-shard cmd block and 40-shard storage block are // not frozen like their files' legacy 20- and 5-shard blocks. // gen_embedded_{cmd,storage}_shard_manifest.py --check verifies only that // the committed block names every discovered test exactly once, failing with @@ -167,7 +167,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { // duration balancing. totalsSet[bazelEmbeddedCmdShardCount(t)] = true case ".github/scripts/embedded-storage-test-shard.sh": - // F1: mirrors the cmd case above for the 20-shard storage block. + // F1: mirrors the cmd case above for the 40-shard storage block. totalsSet[bazelEmbeddedStorageShardCount(t)] = true case ".github/scripts/server-storage-test-shard.sh": totalsSet[bazelServerFullShardCount(t)] = true From 7c47c3c0a96e5adfac6a83a5455f998a60a91526 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 22:11:43 +0000 Subject: [PATCH 024/151] ci(bazel): pin httpclient_served_test's slow tests to shards (16 -> 26 shards) (#7443) After #7440 //internal/httpclient:httpclient_served_test set the embedded lane (149-167s slowest shard in recent CI runs, 63-167s across PRs since #7416). Not a regression: its top-level tests are all serial and rules_go's round-robin put the long ones wherever its index split dropped them, so shard times swung with load and with which tests a PR added (whole-lane rbe-west run on main: 40-119s over 16 shards). The target now runs through tools/bazel/go_test_pinned_shard.sh, as bd_dolt_server_test and regression_test do: every top-level test measured at >= 3s (max over three whole-lane runs) is pinned onto 17 shards (served_pinned_shards.txt, tools/bazel/pin_shards.py), and the binary's own round-robin deals the other ~620 over the remaining 9. Every test still runs in exactly one shard (TestPinnedShardWrapperSplit), the manifest may name only real tests, and the lane's check_testcases.py fails an empty shard. No test changed; the heaviest single test (~52s, no subtests) stays whole. The pinned-wrapper policy listed only dolt-server-cmd targets, since the wrapper selects and skips; pinnedShardTargets now carries each reviewed target's lane tag, and httpclient_served_test is reviewed in as the embedded lane's (its args stay pinned in TestBazelRetiredLanesCannotBeNarrowed). rbe-west, `bazel test //... --config=embedded --nocache_test_results`: served shards max 119s -> 77s, median 73s -> 54s; lane critical path 138-153s -> 116s. Bead: ga-vnycm2.30 Co-authored-by: Claude Opus 5.5 (1M context) --- internal/httpclient/BUILD.bazel | 36 +++-- internal/httpclient/served_pinned_shards.txt | 142 +++++++++++++++++++ scripts/BUILD.bazel | 1 + scripts/conformance_script_test.go | 2 +- scripts/pinned_shards_helpers_test.go | 16 ++- scripts/pinned_shards_test.go | 21 +-- scripts/pr_risk_bazel_coverage_test.go | 9 +- 7 files changed, 199 insertions(+), 28 deletions(-) create mode 100644 internal/httpclient/served_pinned_shards.txt diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 1f4e4a6ad..a2b9d9346 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -207,14 +207,26 @@ go_test( # fails each served case rather than skipping it. It is the script's run on # the lane's race build (raceEnabled shrinks its one large apply). # -# Shard count: on the lane's executors (rbe-west, 2026-10-09) the package is -# 725 tests and ~1,300 test-seconds, the heaviest about 54s. Each served case -# copies one per-process template workspace instead of initializing its own -# embedded Dolt schema (served_harness_test.go's servedTemplateWorkspace), -# and TestServedBatchGetterContract seeds its 1000 issues in one write: before +# Shard count: on the lane's executors (rbe-west, 2026-10-09, the whole +# embedded lane running) the package is ~750 top-level tests and ~1,100 +# test-seconds, all serial, the heaviest about 52s. Each served case copies +# one per-process template workspace instead of initializing its own embedded +# Dolt schema (served_harness_test.go's servedTemplateWorkspace), and +# TestServedBatchGetterContract seeds its 1000 issues in one write: before # both, the package was ~3,700 test-seconds over 24 shards, the heaviest shard -# 390s. rules_go deals the tests out round-robin; at 16 shards the measured -# loads come to 62-112s per shard. +# 390s. rules_go's round-robin alone (16 shards) left shards at 40-119s, +# wherever its split dropped the long tests, so the tests measured at >= 3s +# (the maximum over three whole-lane runs) are pinned +# (tools/bazel/go_test_pinned_shard.sh, served_pinned_shards.txt from +# tools/bazel/pin_shards.py --shards 17 --min-seconds 3) onto 17 shards, and +# the binary's round-robin deals the other ~650 (~340s of measured test +# time; those shards ran ~1.4x their tests' summed times) over the remaining +# 9. +# Regenerate the manifest from runs of the lane: +# bazel test //... --config=embedded +# python3 tools/bazel/pin_shards.py --shards 17 --min-seconds 3 \ +# --write internal/httpclient/served_pinned_shards.txt \ +# bazel-testlogs/internal/httpclient/httpclient_served_test # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, @@ -227,19 +239,23 @@ sh_test( # that SIGTERM, so a hang would end as a bare TIMEOUT. 19m lets Go's own # timeout fire first and dump every goroutine to test.log. timeout = "eternal", - srcs = ["//tools/bazel:go_test_variant.sh"], + srcs = ["//tools/bazel:go_test_pinned_shard.sh"], args = [ + "$(rootpath :served_pinned_shards.txt)", "$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m", ], - data = [":httpclient_test"], + data = [ + "served_pinned_shards.txt", + ":httpclient_test", + ], env = { "BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1", }, - shard_count = 16, + shard_count = 26, tags = ["embedded"], ) diff --git a/internal/httpclient/served_pinned_shards.txt b/internal/httpclient/served_pinned_shards.txt new file mode 100644 index 000000000..a62da7437 --- /dev/null +++ b/internal/httpclient/served_pinned_shards.txt @@ -0,0 +1,142 @@ +# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by +# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage). +# +# Every top-level test measured at >= 3s, bin-packed +# longest-first onto these shards; every other test runs in the +# binary's own round-robin shards. Measured seconds per pinned shard: +# 86 88 85 89 86 86 86 86 86 86 86 86 86 89 89 86 86 +# Unpinned tests: 617, 335s in total. +# +# +1 TestServedBatchApplyEndGateCycleSurvivesSkipPerEdgeCycleCheck +1 TestServedBatchApplyUpdateAfterCloseInOneRequest +1 TestServedDualWriteContract +1 TestServedIssueOperationsCreateUnderAParentMintsTheNextChildID +1 TestServedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix +2 TestServedBatchApplyShape356CommitsAtomically +2 TestServedBatchGetterContract +2 TestServedDetailReadsAgreeWithTheReferenceStore +2 TestServedLifecycleCloseAdmitsATransitivelyBlockedTarget +2 TestServedReaderListAppliesTheClientSideDisplayOrder +2 TestServedReaderListStatusORSetUnderALabelScope +2 TestServedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary +3 TestServedBatchApplyAllowsAClosedParentToGainAnOpenChild +3 TestServedBatchApplyBoundsTheItemCount +3 TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows +3 TestServedCycleDetectorContract +3 TestServedLeaseReclaimerScopesToAssigneesAndLabels +3 TestServedLifecycleCloseIsIdempotentAndKeepsTheFirstClose +3 TestServedReaderListMaxRowsUnderPushdown +4 TestServedBatchApplyANoOpBatchRecordsNoHistory +4 TestServedBatchApplyAppliesEveryItemInDeclarationOrder +4 TestServedBatchApplyRefusesExpectedVersionOnARowAnEarlierItemTouched +4 TestServedDeleteGuardsADurableNamedWithAWispDependent +4 TestServedDeleterCountsCrossPlaneEdgesItRemoves +4 TestServedDeleterSettlesTheSurvivorsOfADeletedBlocker +4 TestServedLifecycleReopenRecordsItsReason +4 TestServedReaderListOrderMatchesTheReferenceStore +5 TestServedBatchApplyAnEphemeralBatchKeepsItsWispsAndRecordsNoDurableHistory +5 TestServedBatchApplyEndGateRefusesAHierarchyTheRequestBuilt +5 TestServedBatchApplyRefusesAKeyDeclaredLater +5 TestServedLifecycleUpdateClosePolicy +5 TestServedLifecycleUpdateRefusesATemplate +5 TestServedReaderListMaxRowsBoundsWireRowsFetched +5 TestServedReadsAndWritesRefuseAfterAPostHandshakeServerSwap +6 TestServedBatchApplyRefusesARefNamingNeitherOrBoth +6 TestServedBatchApplyRefusesExpectedVersionOnARowAnEarlierItemCreated +6 TestServedDeleteGuardsAWispNamedWithADurableDependent +6 TestServedDeleterSettlesTheSurvivorsOfADeletedWispBlocker +6 TestServedLifecycleCloseRefusalsCarryTheirTypesAndWriteNothing +6 TestServedLifecycleCloseSettlesTheClosedRowItselfAndItsChild +6 TestServedReaderListSortPushdownIsOneDial +6 TestServedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded +7 TestServedBatchApplyARefusedRequestRecordsNoHistory +7 TestServedBatchApplyAnswersEveryItemWithoutASnapshot +7 TestServedBatchApplyClosePolicyEvaluatesAtTheCloseItem +7 TestServedBatchApplySplicesAForwardMetadataRef +7 TestServedBatchCloserSettlesTheDependersOfWhatItClosed +7 TestServedLifecycleCloseCountsOpenChildrenInBothPlanes +7 TestServedQuerierDisjunctionAnswersEveryMatch +7 TestServedWorkspaceConfigContract +8 TestDualRunCloseHistoryAgreesWithTheReferenceStore +8 TestServedBatchApplyBindsEachNamedKeyToItsMintedID +8 TestServedBatchApplyEvaluatesExpectedAssigneeAsModified +8 TestServedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate +8 TestServedGraphCounterContract +8 TestServedIssueOperationsCreateParentChildRecomputesWaitsForClosure +8 TestServedLifecycleCloseSettlesItsTransitiveAndCrossPlaneDependers +8 TestServedQuerierSortTieBreaksByIDInBothDirections +9 TestDualRunReaderAgreesWithTheReferenceStore +9 TestServedBatchApplyResolvesABackwardKeyRef +9 TestServedDependencyEditorMaintainsBlockedStateAcrossPlanes +9 TestServedDependencyEditorRecordsOneHistoryEntryPerLandedRequest +9 TestServedLifecycleCloseEnforcesTheCloseGuards +9 TestServedLifecycleCloseOnASpawnersLastChildSatisfiesAWaitsForGate +9 TestServedReaderListWeldedOrderKeepsSendingNoSortParameter +9 TestServedReaderRefusesTheFiltersTheWireCannotCarry +10 TestServedBatchApplyEvaluatesExpectedStatusAsModified +10 TestServedBatchCloserForceBypassesOnlyClosePolicy +10 TestServedBlockingAnnotatorContract +10 TestServedCounterContract +10 TestServedDependencyEditorRemoveUnmarksItsSourceAndDescendants +10 TestServedQuerierSortSeesTheWholeMatchingSet +10 TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope +10 TestServedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven +11 TestServedBatchApplyExpectedVersionThatMatchesLetsTheItemThrough +11 TestServedBatchApplyNeverReordersItsItems +11 TestServedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit +11 TestServedRawCloseKeepsParityWithTheOtherBackends +11 TestServedReaderListKeysetPositionSkipsForward +11 TestServedReaderListLimitBoundaryUnderALabelScope +11 TestServedTreeWalkerContract +11 TestTheParentWalkAnswersTheSameRowsOverHTTP +12 TestDualRunIDResolutionProbesAgreeWithTheReferenceStore +12 TestServedBatchApplyRecordsOneEntryForAWriteThatLandedNothing +12 TestServedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction +12 TestServedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender +12 TestServedReaderDoesNotMutateTheCallerRequestOverTheWire +12 TestServedReaderReadyParentScopesToItsTransitiveDescendants +12 TestTheCompositionCatchesAWrongParameter +12 TestTheReadyBridgeDualRunsBuildReadyFilterOverBothReadPaths +13 TestServedBatchApplyNormalizesTheWaitsForGate +13 TestServedBatchApplyRefusesACrossPlaneEdgeBetweenRowsItCreated +13 TestServedIssueOperationsUpdateClaimConflictCarriesTheLosingState +13 TestServedIssueOperationsUpdateStatusCrossingSettlesDependers +13 TestServedJournalContract +13 TestServedReaderListUnlimitedStaysAWalk +13 TestServedReaderListWalksMultiplePages +13 TestServedRelationsContract +14 TestServedBatchApplyRefusesAKeyNoItemDeclares +14 TestServedBatchApplyRollsBackEverythingWhenTheLastItemRefuses +14 TestServedLifecycleReopenReblocksItsDependers +14 TestServedLifecycleUpdateParentIDReplacesTheParentEdge +14 TestServedQuerierPageIsAPrefixAndHasMoreIsExact +14 TestServedReaderListMessageWispsNeedTheInfraIntentAndNotJustThePlane +14 TestServedReaderListMultiPageWalkHasNoSnapshotIsolation +14 TestServedSweeperProtectsLiveDependents +14 TestTheParentWalkRefusesAResidualOverHTTP +15 TestDualRunQuerierAndReadyCounterAgreeWithTheReferenceStore +15 TestServedBatchApplyStaleExpectedVersionRefusesTheWholeRequest +15 TestServedDependencyEditorRefusesACycleThroughAParentChildHop +15 TestServedLifecycleUpdateAssigneeTransferFence +15 TestServedLifecycleUpdateParentIDReplacesEveryParent +15 TestServedQuerierSortBoundsThePageInOrder +15 TestServedReaderListEmptyPageIsWellFormedUnderALabelScope +15 TestServedReaderListFallsBackToTheWalkWithoutTheCapability +15 TestServedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities +16 TestServedBatchApplyAcceptsAnExternalEdgeTarget +16 TestServedBatchApplyHistoryNamesTheActorAndReadsTheProvenance +16 TestServedDeleterSettlesTheChildrenOfADeletedParent +16 TestServedEdgeReaderContract +16 TestServedQuerierSortByClosedPutsTheUnclosedRowsAtTheFarEnd +16 TestServedQuerierSortByTitleFoldsCaseBeforeItCutsThePage +16 TestServedReaderListPriorityKeysetPositionResumesThePriorityOrder +16 TestServedSweeperProtectsLiveDependentsAcrossPlanes +17 TestServedBatchApplyRecordsExactlyOneHistoryEntry +17 TestServedBatchApplyReportsChangedPerItem +17 TestServedBatchCloserIdempotentRecloseIsAPerItemSuccess +17 TestServedCommenterContract +17 TestServedLifecycleCreateInheritsParentLabels +17 TestServedQuerierHidesClosedUnlessTheExpressionOrTheFlagSaysOtherwise +17 TestServedReaderReadyDeferredAndEphemeralGates +17 TestServedStatsReporterContract diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 36b26accd..a91b79346 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -160,6 +160,7 @@ go_test( "//:nested_go_modules", "//:repo_go_test_srcs", "//cmd/bd:repo_other_files", + "//internal/httpclient:repo_other_files", "//internal/storage/dolt:repo_other_files", "//internal/storage/embeddeddolt:repo_other_files", "//tests/regression:repo_other_files", diff --git a/scripts/conformance_script_test.go b/scripts/conformance_script_test.go index ce41f1067..24fd6ebab 100644 --- a/scripts/conformance_script_test.go +++ b/scripts/conformance_script_test.go @@ -123,7 +123,7 @@ func TestConformanceServedTierRunsUnderBazel(t *testing.T) { if rule == "" { t.Fatal("internal/httpclient/BUILD.bazel has no httpclient_served_test") } - want := []string{`"$(rootpath :httpclient_test)",`, `data = [":httpclient_test"],`, `tags = ["embedded"],`} + want := []string{`"$(rootpath :httpclient_test)",`, `":httpclient_test",`, `tags = ["embedded"],`} for _, sw := range switches { want = append(want, `"`+sw[1]+`": "`+sw[2]+`",`) } diff --git a/scripts/pinned_shards_helpers_test.go b/scripts/pinned_shards_helpers_test.go index 862fcd0b4..52476fbb5 100644 --- a/scripts/pinned_shards_helpers_test.go +++ b/scripts/pinned_shards_helpers_test.go @@ -16,12 +16,20 @@ const pinnedShardWrapper = "tools/bazel/go_test_pinned_shard.sh" // pinnedShardTargets are the sh_tests that run a go_test binary through // tools/bazel/go_test_pinned_shard.sh: the BUILD file, the rule, its -// manifest, and the directory holding the go_test's sources. +// manifest, the directory holding the go_test's sources, and the rule's one +// lane tag. +// +// httpclient_served_test is the embedded lane's (a retired tier's) served +// corpus, which may not be narrowed: the wrapper runs every top-level test in +// exactly one shard (TestPinnedShardWrapperSplit), the manifest may name only +// real tests (TestPinnedShardManifestsNameRealTests), and the lane's +// check_testcases.py fails a shard that ran none. var pinnedShardTargets = []struct { - build, rule, manifest, pkg string + build, rule, manifest, pkg, tag string }{ - {"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd"}, - {"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression"}, + {"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd", "dolt-server-cmd"}, + {"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression", "dolt-server-cmd"}, + {"internal/httpclient/BUILD.bazel", "httpclient_served_test", "internal/httpclient/served_pinned_shards.txt", "internal/httpclient", "embedded"}, } var pinnedLineRe = regexp.MustCompile(`^([1-9][0-9]*) (Test[A-Za-z0-9_]*)$`) diff --git a/scripts/pinned_shards_test.go b/scripts/pinned_shards_test.go index ff95e9f3f..765415979 100644 --- a/scripts/pinned_shards_test.go +++ b/scripts/pinned_shards_test.go @@ -211,14 +211,15 @@ func TestPinShardsGenerator(t *testing.T) { // pinnedShardWrapperUsers returns an error for every BUILD rule that runs // through go_test_pinned_shard.sh other than pinnedShardTargets, and for any -// of those tagged for another lane than the cmd/bd Dolt-server tier: the -// wrapper selects and skips tests, which the retired tiers' lanes forbid -// (TestBazelRetiredLanesCannotBeNarrowed). +// of those not tagged exactly its pinnedShardTargets lane: the wrapper +// selects and skips tests, which the retired tiers' lanes forbid +// (TestBazelRetiredLanesCannotBeNarrowed) except for the reviewed targets +// listed there. func pinnedShardWrapperUsers(t *testing.T, root string) []string { t.Helper() - allowed := map[string]bool{} + allowed := map[string]string{} for _, tgt := range pinnedShardTargets { - allowed[tgt.build+":"+tgt.rule] = true + allowed[tgt.build+":"+tgt.rule] = tgt.tag } const label = "//tools/bazel:go_test_pinned_shard.sh" var errs []string @@ -247,11 +248,13 @@ func pinnedShardWrapperUsers(t *testing.T, root string) []string { continue } seen++ - if !allowed[f+":"+name] { - errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+"; add it to pinnedShardTargets after reviewing that it is not a retired tier's lane") + tag, ok := allowed[f+":"+name] + if !ok { + errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+"; add it to pinnedShardTargets after reviewing that every test still runs exactly once") + continue } - if !strings.Contains(rule, `tags = ["dolt-server-cmd"]`) { - errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+" but is not tagged exactly dolt-server-cmd") + if !strings.Contains(rule, `tags = ["`+tag+`"]`) { + errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+" but is not tagged exactly "+tag) } } } diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index d1e800aec..0a57f6785 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -811,9 +811,10 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { } // go_test_pinned_shard.sh selects and skips by design (each test in // exactly one shard: TestPinnedShardWrapperSplit), and is reviewed - // here for the dolt-server-cmd targets only, which are none of the - // retired tiers' lanes; pinnedShardWrapperUsers fails if anything - // else runs through it. + // here for pinnedShardTargets only (the dolt-server-cmd targets and + // the embedded lane's httpclient_served_test, whose args are pinned + // above); pinnedShardWrapperUsers fails if anything else runs + // through it. exempt := filepath.ToSlash(rel) == pinnedShardWrapper if exempt { for _, e := range pinnedShardWrapperUsers(t, root) { @@ -890,7 +891,7 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { // script's switches: required, so a missing engine fails each served // case instead of skipping it. "//internal/httpclient:httpclient_served_test": { - []string{"$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, + []string{"$(rootpath :served_pinned_shards.txt)", "$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1"}, }, "//cmd/bd:bd_proxied_test": { From d15958a5eb88fc32e3a47680ccc017ff47b561ac Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 22:56:46 +0000 Subject: [PATCH 025/151] fix(issueops): enforce the dotted-child dependency rule on every leg (#7424) * fix(issueops): enforce the dotted-child dependency rule on every leg The refusal of an explicit edge from a dotted-id child to its own ancestor (bd-abc.1 -> bd-abc; only the parent-child edge to the immediate dotted parent is allowed) lived only in cmd/bd (isDisallowedHierarchicalDependency). HTTP dependencies:add, batch:apply dep_add items and `bd create --graph`'s own edges skipped it. It is now one library rule, issueops.CheckDottedChildDependency, returning *issueops.DottedChildDependencyError (unwraps to ErrValidation, same message the CLI printed): - ValidateAddDependenciesRequest applies it, so DependencyEditor refuses on dolt, embedded, uow and the server behind httpapi (400 invalid_argument); - both BatchApplier bodies (issueops and uow) apply it to each dep_add item on the resolved ids, so a key bound to a minted dotted child is refused too; - the HTTP client checks it before the dial, as it does self-dependency, so the typed error survives the wire. cmd/bd keeps no copy: single-edge dep add / dep --blocks rely on the role; bd link (still on the store verb) and the --file bulk reporter call the library rule. CLI output is byte-identical (checked against origin/main). Conformance: RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent and RunBatchApplyRefusesADottedChildGatedOnItsOwnParent wired on dolt, embedded, uow and http, plus a raw-wire served test past the client check and handler mapping cases. Co-Authored-By: Claude Opus 5.5 * fix(batch): refuse a dotted child gated on its own ancestor in bd batch Review fixup for PR #7424 (iterations 1 and 2). Major: a bd batch script line `dep add bd-abc.1 bd-abc` still wrote the edge this PR refuses everywhere else, on both backends. Neither batch dispatch writes through the DependencyEditor role that enforces the dotted-child rule: the classic arm calls storage.Transaction.AddDependency and the proxied arm the unit of work's AddDependencies verb. Both arms now build the edge with one helper, batchDepAddEdge, which keeps the default type and the arity and type errors, and asks publicops.CheckDottedChildDependency the way bd link does. A refused line rolls the whole batch back. Tests: TestBatchDepAddEdge (default and explicit types, arity and type errors, three refusals typed *DottedChildDependencyError matching ErrValidation, and the immediate parent-child edge allowed), and TestProxiedServerBatchDepAddRefusesADottedChildGatedOnItsParent (classic and proxied: the refusal names line 2, rolls back an earlier update line and writes no edge, and `dep add C P parent-child` still lands exactly one), registered in the proxied shard manifest. A mutant that drops the rule from the shared helper fails both; one that bypasses the helper in the proxied arm alone fails the proxied leg. Minors: - For a child whose parent-child edge is already stored (a `bd create --parent` child) the rule answers ahead of the hierarchy and type conflicts, so an HTTP caller now gets 400 invalid_argument where it got 409 dependency_cycle (blocking types) or dependency_exists (any other type). Documented in the DependencyEditor role doc, the CHANGELOG entry and the httpapi batch-apply test comment, and pinned by a stored-hierarchy phase in the DependencyEditor and BatchApplier dotted-child conformance contracts on every leg; a mutant that drops the rule gets the hierarchy conflict instead (409 dependency_cycle on the wire). - The bd create --graph dry-run note now says the live create may also refuse an edge that gates a dotted-id child on its own ancestor. - The DottedChildDependencyError comment says the message is kept verbatim and says "deadlock" whatever the type. Not changed: bd relate (relates-to does not schedule) and the edges a create writes with its issue (bd create --deps/--waits-for, and over HTTP an explicit dotted id with a non-immediate ParentID) still write through paths that skip the rule. The scheduling shapes need an explicit dotted id or a grandparent target. Follow-up bd-25s1wh tracks moving the rule down to the domain/store add verbs, with a structural bypass for reparent and template edges. Iteration 2 (minor): the CheckDottedChildDependency doc no longer implies bd link and bd batch are the only writers outside the roles. It names bd relate and the create edges as not applying the rule yet and cites bd-25s1wh. Complexity (cyclomatic/cognitive): runBatchOp 23/35 -> 21/31, runBatchOpUOW 28/48 -> 26/44, batchDepAddEdge 5/4. Touching them brings both pre-existing dispatch switches into the measured set; splitting them is out of scope here. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * docs(issueops,httpapi): document the dotted-child 400 and widen its gap list Review fixup for PR #7424 (relaunch, iteration 1, minors). - The OpenAPI spec now describes the wire change this PR makes. The dependencies:add and issues:batchApply 400 vocabularies name an edge from a dotted-id child to its own ancestor (anything but the parent-child edge to its immediate parent; param `edges` and `items`). dependencies:add says why that is request-intrinsic, and both 409s say it is never dependency_cycle or dependency_exists, even for a child whose parent-child edge is stored. Response descriptions only: the generated types and the wireshape digest are unchanged (make api-check). - The CheckDottedChildDependency doc no longer reads as a complete list of the writers that skip the rule. Besides bd relate and the create edges it names bd duplicate, bd supersede, bd duplicates --auto-merge (its related link and its reparenting) and the tracker engine's createDependencies, and says the molecule, template and swarm writers are unaudited. bd-25s1wh's notes carry the widened scope, including `bd duplicate bd-a.1 --of bd-a`. Declined (nit): unifying the root issueops import alias across the PR's cmd/bd files. batch.go is the one file that imports both issueops packages, so one of them needs an alias there; it uses publicops, the repo's usual alias for the root package, and aliasing the other side instead would rewrite base-owned lines. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../scripts/embedded-storage-test-shards.txt | 1 + .github/scripts/proxied-cmd-test-shards.txt | 1 + CHANGELOG.md | 16 ++++ backend/conformance/batch_apply_contract.go | 77 ++++++++++++++++ .../conformance/dependency_editor_contract.go | 79 ++++++++++++++++ backend/conformance/role_bundle_cases.go | 2 + cmd/bd/BUILD.bazel | 1 - cmd/bd/batch.go | 54 +++++++---- cmd/bd/batch_parity_test.go | 71 +++++++++++++++ cmd/bd/batch_proxied_server.go | 21 +---- cmd/bd/batch_test.go | 89 ++++++++++++++++++ cmd/bd/dep.go | 48 +--------- cmd/bd/dep_proxied_server.go | 9 +- cmd/bd/dep_test.go | 82 ----------------- cmd/bd/dependency_hierarchy_test.go | 55 ----------- cmd/bd/graph_apply.go | 2 +- cmd/bd/link.go | 7 +- cmd/bd/link_proxied_server.go | 5 +- internal/httpapi/batch_apply_test.go | 13 +++ internal/httpapi/dependency_edit_test.go | 14 +++ internal/httpapi/spec/openapi.v0.yaml | 24 ++++- internal/httpclient/dependencyeditor.go | 6 +- .../httpclient/served_batch_apply_test.go | 4 + .../httpclient/served_memories_deps_test.go | 39 ++++++++ .../storage/dolt/batch_apply_contract_test.go | 5 + .../dolt/dependency_editor_contract_test.go | 6 ++ .../embeddeddolt/batch_apply_contract_test.go | 3 + .../dependency_editor_contract_test.go | 6 ++ internal/storage/issueops/batch_apply.go | 6 ++ .../storage/issueops/dependency_editor.go | 9 ++ internal/storage/uow/batch_applier.go | 3 + .../storage/uow/batch_apply_contract_test.go | 3 + .../uow/dependency_editor_contract_test.go | 1 + issueops/dependencyeditor.go | 91 +++++++++++++++++-- issueops/issueops_external_test.go | 55 +++++++++++ 35 files changed, 669 insertions(+), 239 deletions(-) delete mode 100644 cmd/bd/dependency_hierarchy_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 7e9f2991d..8e103b3e0 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -152,6 +152,7 @@ 40 3 TestCrossRecordInvariantContract 40 3 TestDepRekeyMarkerStillInIgnoredSeries 40 3 TestDoltCommitResponseLossIsIndeterminate +40 3 TestEmbeddedDependencyEditorRefusesADottedChildGatedOnItsOwnParent 40 3 TestEmbeddedDoltStoreIssueLifecycleBuildsOperations 40 3 TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore 40 3 TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 42a1fc999..72e73e898 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -392,6 +392,7 @@ 34 20 TestProxiedServerWispGC 34 20 TestServerModeSharedServeGlobalReportsTheServedDatabase +34 21 TestProxiedServerBatchDepAddRefusesADottedChildGatedOnItsParent 34 21 TestProxiedServerConfig 34 21 TestProxiedServerExternalPolicyDirectParity 34 21 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity diff --git a/CHANGELOG.md b/CHANGELOG.md index 230391fdc..5477be6ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -263,6 +263,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `POST /v0/beads/issues/{id}:claim` (`issueops.Claimer`) still claims a template that a `PATCH` with `claim: true` refuses. Close's guards move to the role in their own change (#7425). +- The dotted-id hierarchy refusal (a child such as `bd-abc.1` may not carry an + explicit edge to its own ancestor, other than the parent-child edge to its + immediate parent) moved from `cmd/bd` into the library as + `issueops.CheckDottedChildDependency` / `*issueops.DottedChildDependencyError` + (unwraps to `ErrValidation`). Every `DependencyEditor` (dolt, embedded, + unit of work, HTTP) and every `BatchApplier` dep_add item now enforces it, so + an HTTP `dependencies:add`, a `batch:apply` edge and `bd create --graph`'s own + edges are refused like `bd dep add` (HTTP: 400 `invalid_argument`). `bd dep + add`, `bd dep --blocks`, `bd link` and `bd dep add --file` print the same + messages as before, and a `bd batch` script's `dep add` line, which neither + backend checked, is now refused with that message and rolls the whole batch + back. The refusal is decided from the two ids, ahead of every check that + reads the stored graph, so for a child whose parent-child edge is already + stored (a `bd create --parent` child) an HTTP caller now gets that 400 where + it used to get 409 `dependency_cycle` (a blocking type) or + `dependency_exists` (any other type). - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index c2ac269d9..f6480e47a 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -2657,3 +2657,80 @@ func RunBatchApplyCloseItemsAnswerToTheCloseGuards(t *testing.T, ctx context.Con } } } + +// RunBatchApplyRefusesADottedChildGatedOnItsOwnParent pins the dotted-id +// hierarchy rule (publicops.CheckDottedChildDependency) on a dep_add item: an +// edge from a dotted-id child to its own ancestor is refused with an +// *ItemError naming the edge, and the whole request rolls back. The rule is +// applied to the RESOLVED ids, so a key bound to a dotted child this request +// creates — the `bd create --graph` shape — is refused like a stored id. +// +// POSITIVE HALF: the parent-child edge to the immediate dotted parent lands. +// +// WITH THE HIERARCHY STORED: once that edge exists (the shape `bd create +// --parent` leaves), a blocking edge on the same pair is still this validation +// refusal, not the hierarchy conflict the stored edge would raise (which does +// not match ErrValidation) — the rule reads only the resolved ids, so it +// answers first — and the stored edge survives. +func RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + parent := fixture.IssuePrefix + "-dotted" + child := parent + ".1" + minted := parent + ".2" + batchApplySeedIssue(t, ctx, fixture, parent, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, child, types.StatusOpen) + + for _, refused := range []struct { + name string + items []publicops.ApplyItem + index int + }{ + {"a stored child blocking on its parent", []publicops.ApplyItem{ + batchApplyDepAdd(publicops.Ref{ID: child}, publicops.Ref{ID: parent}, publicops.DepBlocks, ""), + }, 0}, + {"a child this request creates blocking on its parent", []publicops.ApplyItem{ + batchApplyCreate("child", batchApplyIssue(minted, "the new child")), + batchApplyDepAdd(publicops.Ref{Key: "child"}, publicops.Ref{ID: parent}, publicops.DepBlocks, ""), + }, 1}, + } { + _, err := fixture.BatchApplier.ApplyBatch(ctx, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + ForceIDPrefix: true, + Items: refused.items, + }) + if !errors.Is(err, publicops.ErrValidation) { + t.Errorf("%s: error = %v, want ErrValidation", refused.name, err) + continue + } + var itemErr *publicops.ItemError + if !errors.As(err, &itemErr) { + t.Errorf("%s: error = %v, want an *ItemError naming the edge item", refused.name, err) + continue + } + if itemErr.Index != refused.index || itemErr.Kind != publicops.ItemDepAdd { + t.Errorf("%s: ItemError = %#v, want Index %d and Kind %q", refused.name, itemErr, refused.index, publicops.ItemDepAdd) + } + } + assertBatchApplyEdgeCount(t, ctx, fixture, child, parent, 0) + assertBatchApplyRowCount(t, ctx, fixture, "issues", minted, 0) + + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + batchApplyDepAdd(publicops.Ref{ID: child}, publicops.Ref{ID: parent}, publicops.DepParentChild, ""), + }, + }) + assertBatchApplyEdgeCount(t, ctx, fixture, child, parent, 1) + + _, err := fixture.BatchApplier.ApplyBatch(ctx, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + batchApplyDepAdd(publicops.Ref{ID: child}, publicops.Ref{ID: parent}, publicops.DepBlocks, ""), + }, + }) + var itemErr *publicops.ItemError + if !errors.Is(err, publicops.ErrValidation) || !errors.As(err, &itemErr) { + t.Errorf("blocks on a parent whose parent-child edge is stored: error = %v, want an *ItemError matching ErrValidation", err) + } + assertBatchApplyEdgeCount(t, ctx, fixture, child, parent, 1) +} diff --git a/backend/conformance/dependency_editor_contract.go b/backend/conformance/dependency_editor_contract.go index e037e9dbe..4f5b33268 100644 --- a/backend/conformance/dependency_editor_contract.go +++ b/backend/conformance/dependency_editor_contract.go @@ -2631,3 +2631,82 @@ func dependencyEditorHistoryProbe(t *testing.T, ctx context.Context, fixture Dep assertDependencyEditorHistoryDelta(t, ctx, fixture, before, want, why) } } + +// RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent pins +// publicops.CheckDottedChildDependency on the role: an explicit edge from a +// dotted-id child to its own ancestor is refused as a +// *DottedChildDependencyError (ErrValidation) on every leg, and nothing is +// written. (It is decided before any per-edge probe, so the probe-skipped +// spelling, which the wire does not publish, adds nothing here.) It used to be +// enforced only by the CLI ahead of the role, so an HTTP or embedded caller +// could gate bd-abc.1 on bd-abc. +// +// POSITIVE HALF: the hierarchy itself — a parent-child edge to the IMMEDIATE +// dotted parent — is allowed, and so is an edge between siblings. A body that +// refused every edge touching a dotted id would satisfy the refusals alone. +// +// WITH THE HIERARCHY STORED: once that parent-child edge exists (the shape +// `bd create --parent` leaves), a blocking edge on the same pair is still this +// refusal, not the type or hierarchy conflict the stored edge would raise — the +// rule reads nothing, so it answers first — and the stored edge survives. +func RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t *testing.T, ctx context.Context, fixture DependencyEditorFixture) { + t.Helper() + parent := fixture.IssuePrefix + "-dotted" + child := parent + ".1" + grandchild := child + ".2" + sibling := parent + ".3" + for _, id := range []string{parent, child, grandchild, sibling} { + seedDependencyEditorIssue(t, ctx, fixture, id) + } + + for _, refused := range []struct { + name string + edge publicops.DependencyEdge + }{ + {"child blocks on its parent", publicops.DependencyEdge{IssueID: child, DependsOnID: parent, Type: publicops.DepBlocks}}, + {"grandchild blocks on its grandparent", publicops.DependencyEdge{IssueID: grandchild, DependsOnID: parent, Type: publicops.DepBlocks}}, + {"grandchild parent-child to its grandparent", publicops.DependencyEdge{IssueID: grandchild, DependsOnID: parent, Type: publicops.DepParentChild}}, + {"child relates to its parent", publicops.DependencyEdge{IssueID: child, DependsOnID: parent, Type: publicops.DepRelated}}, + } { + _, err := fixture.Editor.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "writer", + Edges: []publicops.DependencyEdge{refused.edge}, + }) + var dotted *publicops.DottedChildDependencyError + if !errors.As(err, &dotted) { + t.Errorf("%s: error = %v, want *DottedChildDependencyError", refused.name, err) + continue + } + if !errors.Is(err, publicops.ErrValidation) { + t.Errorf("%s: error = %v, want it to match ErrValidation", refused.name, err) + } + if dotted.IssueID != refused.edge.IssueID || dotted.DependsOnID != refused.edge.DependsOnID { + t.Errorf("%s: refusal names %s -> %s, want %s -> %s", refused.name, + dotted.IssueID, dotted.DependsOnID, refused.edge.IssueID, refused.edge.DependsOnID) + } + } + assertDependencyEditorNoEdgesFrom(t, ctx, fixture, child, grandchild) + + if _, err := fixture.Editor.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "writer", + Edges: []publicops.DependencyEdge{ + {IssueID: grandchild, DependsOnID: child, Type: publicops.DepParentChild}, + {IssueID: sibling, DependsOnID: child, Type: publicops.DepBlocks}, + }, + }); err != nil { + t.Fatalf("the immediate parent-child edge and a sibling edge: %v", err) + } + assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", grandchild, child, string(publicops.DepParentChild), 1) + assertDependencyEdgeCount(t, ctx, fixture, "dependencies", sibling, child, 1) + + _, err := fixture.Editor.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "writer", + Edges: []publicops.DependencyEdge{{IssueID: grandchild, DependsOnID: child, Type: publicops.DepBlocks}}, + }) + var dotted *publicops.DottedChildDependencyError + if !errors.As(err, &dotted) || !errors.Is(err, publicops.ErrValidation) { + t.Errorf("blocks on a parent whose parent-child edge is stored: error = %v, want *DottedChildDependencyError matching ErrValidation", err) + } + assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", grandchild, child, string(publicops.DepParentChild), 1) + assertDependencyEdgeCount(t, ctx, fixture, "dependencies", grandchild, child, 1) +} diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 27f0d762f..0555b19a5 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -61,6 +61,7 @@ var roleContractCases = []roleContract{ RunBatchApplyUpdateItemsRefuseATemplate, RunBatchApplySplicesTheMetadataOfATemplateItCreates, RunBatchApplyCloseItemsAnswerToTheCloseGuards, + RunBatchApplyRefusesADottedChildGatedOnItsOwnParent, ), roleCases("BatchCloser", "BatchCloser()", oncePerRole, @@ -301,6 +302,7 @@ var roleContractCases = []roleContract{ RunDependencyEditorAcceptsADiamond, RunDependencyEditorGateScopeFollowsTheEdgeType, RunDependencyEditorAcceptsBlockingAcrossIssueTypes, + RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent, ), // The accessor named here is not an accessor at all, alone among these diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index d808cbd5d..7d1742cef 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -676,7 +676,6 @@ go_test( "dep_test.go", "dep_warning_test.go", "dependency_hierarchy_embedded_test.go", - "dependency_hierarchy_test.go", "diff_embedded_test.go", "directory_flag_test.go", "doctor_backend_guard_test.go", diff --git a/cmd/bd/batch.go b/cmd/bd/batch.go index 01c7898bb..99e8b76e1 100644 --- a/cmd/bd/batch.go +++ b/cmd/bd/batch.go @@ -15,6 +15,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // Scope note: @@ -417,27 +418,14 @@ func runBatchOp(ctx context.Context, tx storage.Transaction, op batchOp) (batchO return result, nil case "dep.add": - if len(op.args) < 2 { - return result, fmt.Errorf("dep add requires ") - } - from, to := op.args[0], op.args[1] - depType := "blocks" - if len(op.args) >= 3 { - depType = op.args[2] - } - dt := types.DependencyType(depType) - if !dt.IsValid() { - return result, fmt.Errorf("dep add: invalid dependency type %q", depType) - } - dep := &types.Dependency{ - IssueID: from, - DependsOnID: to, - Type: dt, + dep, err := batchDepAddEdge(op.args) + if err != nil { + return result, err } if err := tx.AddDependency(ctx, dep, actorName); err != nil { return result, err } - result.Target = fmt.Sprintf("%s->%s", from, to) + result.Target = fmt.Sprintf("%s->%s", dep.IssueID, dep.DependsOnID) return result, nil case "dep.remove": @@ -454,6 +442,38 @@ func runBatchOp(ctx context.Context, tx storage.Transaction, op batchOp) (batchO return result, fmt.Errorf("internal: unhandled batch op %q", op.cmd) } +// batchDepAddEdge builds the edge a `dep add [type]` line +// asks for. Both backends' dispatch switches call it, so the default type, the +// arity and type errors, and the dotted-id hierarchy rule cannot drift between +// them. +// Neither backend writes the edge through the DependencyEditor role that +// enforces that rule (storage.Transaction.AddDependency and the unit of work's +// AddDependencies verb do not), so batch asks the role's rule directly, the +// way bd link does: a script line `dep add bd-abc.1 bd-abc` is refused, and +// rolls the batch back, exactly as `bd dep add bd-abc.1 bd-abc` is. +func batchDepAddEdge(args []string) (*types.Dependency, error) { + if len(args) < 2 { + return nil, fmt.Errorf("dep add requires ") + } + from, to := args[0], args[1] + depType := "blocks" + if len(args) >= 3 { + depType = args[2] + } + dt := types.DependencyType(depType) + if !dt.IsValid() { + return nil, fmt.Errorf("dep add: invalid dependency type %q", depType) + } + if err := publicops.CheckDottedChildDependency(from, to, dt); err != nil { + return nil, err + } + return &types.Dependency{ + IssueID: from, + DependsOnID: to, + Type: dt, + }, nil +} + // parseUpdateKVs walks a slice of "key=value" tokens and builds the updates // map accepted by storage.Transaction.UpdateIssue. Only a small, documented // subset of fields is allowed — anything else is a hard error so typos in diff --git a/cmd/bd/batch_parity_test.go b/cmd/bd/batch_parity_test.go index f150c845b..d50c4d171 100644 --- a/cmd/bd/batch_parity_test.go +++ b/cmd/bd/batch_parity_test.go @@ -259,6 +259,77 @@ func assertBatchParity(t *testing.T, classic, proxied batchParityOutcome) { } } +// TestProxiedServerBatchDepAddRefusesADottedChildGatedOnItsParent pins the +// dotted-id hierarchy rule on a batched `dep add`. Neither backend writes that +// edge through the DependencyEditor role that enforces the rule, so batch asks +// the rule itself (batchDepAddEdge). Both backends must refuse bd-abc.1 -> +// bd-abc with the sentence `bd dep add` prints, name the line, and roll the +// whole batch back, while the one edge the rule allows — parent-child to the +// immediate parent — still lands. +func TestProxiedServerBatchDepAddRefusesADottedChildGatedOnItsParent(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + type outcome struct { + refusedCode int + refusalReported bool + rolledBack bool + edgesAfterRefusal int + allowedCode int + edgesAfterAllowed int + } + outcomes := make(map[string]outcome, 2) + + for _, env := range newCrossModeEnvs(t, bd, "bdc", "bdp") { + var got outcome + + parent := env.create(t, "Dotted parent") + // An explicit dotted id stores no parent-child edge (only --parent + // does), so every edge counted below is one a batch wrote. + child := env.create(t, "Dotted child", "--id", parent+".1", "--priority", "2") + + // Line 1 is a valid write the refusal on line 2 must take back with it. + script := fmt.Sprintf("update %s priority=1\ndep add %s %s\n", child, child, parent) + stdout, stderr, code := env.runStdin(t, script, "batch") + // The failing line is named the way any failing line is, around the + // sentence `bd dep add` prints for the same edge. + refusal := fmt.Sprintf("line 2 (dep add %s %s): cannot add dependency: %s is already a child of %s.", + child, parent, child, parent) + got.refusedCode = code + got.refusalReported = strings.Contains(stdout+stderr, refusal) + got.rolledBack = env.show(t, child).Priority == 2 + got.edgesAfterRefusal = len(env.depList(t, child)) + if got.refusedCode == 0 || !got.refusalReported { + t.Errorf("[%s] batch exit=%d, want non-zero and %q\nstdout:\n%s\nstderr:\n%s", + env.mode, code, refusal, stdout, stderr) + } + + stdout, stderr, code = env.runStdin(t, fmt.Sprintf("dep add %s %s parent-child\n", child, parent), "batch") + got.allowedCode = code + if code != 0 { + t.Fatalf("[%s] batch parent-child dep add failed with exit %d\nstdout:\n%s\nstderr:\n%s", env.mode, code, stdout, stderr) + } + got.edgesAfterAllowed = len(env.depList(t, child)) + + outcomes[env.mode] = got + + if !got.rolledBack { + t.Errorf("[%s] the refused line did not roll back line 1's priority write", env.mode) + } + if got.edgesAfterRefusal != 0 { + t.Errorf("[%s] the refused batch left %d edges on the child, want 0", env.mode, got.edgesAfterRefusal) + } + if got.edgesAfterAllowed != 1 { + t.Errorf("[%s] batch parent-child dep add: %d edges on the child, want 1", env.mode, got.edgesAfterAllowed) + } + } + + if outcomes["classic"] != outcomes["proxied"] { + t.Errorf("cross-mode divergence:\nclassic=%+v\nproxied=%+v", outcomes["classic"], outcomes["proxied"]) + } +} + // depList reads the dependency records of one issue as a flat array, the shape // `bd dep list --json` documents. func (e crossModeEnv) depList(t *testing.T, id string) []map[string]any { diff --git a/cmd/bd/batch_proxied_server.go b/cmd/bd/batch_proxied_server.go index 99ccb890e..cb666214d 100644 --- a/cmd/bd/batch_proxied_server.go +++ b/cmd/bd/batch_proxied_server.go @@ -161,22 +161,9 @@ func runBatchOpUOW(ctx context.Context, uw uow.UnitOfWork, op batchOp) (batchOpR return result, nil case "dep.add": - if len(op.args) < 2 { - return result, fmt.Errorf("dep add requires ") - } - from, to := op.args[0], op.args[1] - depType := "blocks" - if len(op.args) >= 3 { - depType = op.args[2] - } - dt := types.DependencyType(depType) - if !dt.IsValid() { - return result, fmt.Errorf("dep add: invalid dependency type %q", depType) - } - dep := &types.Dependency{ - IssueID: from, - DependsOnID: to, - Type: dt, + dep, err := batchDepAddEdge(op.args) + if err != nil { + return result, err } // AddDependencies (not the single-edge AddDependency) because it lands // each edge in the plane its own SOURCE lives in, which is the routing @@ -198,7 +185,7 @@ func runBatchOpUOW(ctx context.Context, uw uow.UnitOfWork, op batchOp) (batchOpR if _, err := uw.DependencyUseCase().AddDependencies(ctx, []*types.Dependency{dep}, actorName, domain.BulkAddDepsOpts{}); err != nil { return result, err } - result.Target = fmt.Sprintf("%s->%s", from, to) + result.Target = fmt.Sprintf("%s->%s", dep.IssueID, dep.DependsOnID) return result, nil case "dep.remove": diff --git a/cmd/bd/batch_test.go b/cmd/bd/batch_test.go index 894c29010..834bd49f9 100644 --- a/cmd/bd/batch_test.go +++ b/cmd/bd/batch_test.go @@ -1,9 +1,13 @@ package main import ( + "errors" "reflect" "strings" "testing" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) func TestTokenizeBatchLine(t *testing.T) { @@ -261,6 +265,91 @@ func TestParseUpdateKVs(t *testing.T) { } } +// TestBatchDepAddEdge pins the edge a `dep add` line builds on both backends, +// and that the line asks the dotted-id hierarchy rule before either backend +// writes: the refusal is the role's own typed error, so a batch prints the +// sentence `bd dep add` prints and rolls back on it. +func TestBatchDepAddEdge(t *testing.T) { + tooLong := strings.Repeat("x", types.MaxDependencyTypeLen+1) + tests := []struct { + name string + args []string + want *types.Dependency + wantErr string + wantDotted bool + }{ + { + name: "default type is blocks", + args: []string{"bd-1", "bd-2"}, + want: &types.Dependency{IssueID: "bd-1", DependsOnID: "bd-2", Type: types.DepBlocks}, + }, + { + name: "explicit type", + args: []string{"bd-1", "bd-2", "related"}, + want: &types.Dependency{IssueID: "bd-1", DependsOnID: "bd-2", Type: types.DepRelated}, + }, + { + name: "missing to-id", + args: []string{"bd-1"}, + wantErr: "dep add requires ", + }, + { + name: "invalid type", + args: []string{"bd-1", "bd-2", tooLong}, + wantErr: `dep add: invalid dependency type "` + tooLong + `"`, + }, + { + name: "dotted child gated on its own parent", + args: []string{"bd-abc.1", "bd-abc"}, + wantDotted: true, + }, + { + name: "any explicit type up the tree, not only blocking ones", + args: []string{"bd-abc.1", "bd-abc", "related"}, + wantDotted: true, + }, + { + name: "parent-child past the immediate parent", + args: []string{"bd-abc.1.2", "bd-abc", "parent-child"}, + wantDotted: true, + }, + { + name: "parent-child to the immediate parent is the hierarchy itself", + args: []string{"bd-abc.1", "bd-abc", "parent-child"}, + want: &types.Dependency{IssueID: "bd-abc.1", DependsOnID: "bd-abc", Type: types.DepParentChild}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := batchDepAddEdge(tt.args) + switch { + case tt.wantDotted: + var dotted *publicops.DottedChildDependencyError + if !errors.As(err, &dotted) { + t.Fatalf("batchDepAddEdge(%q) = %+v, %v; want *DottedChildDependencyError", tt.args, got, err) + } + if !errors.Is(err, publicops.ErrValidation) { + t.Errorf("refusal %v does not match ErrValidation", err) + } + if dotted.IssueID != tt.args[0] || dotted.DependsOnID != tt.args[1] { + t.Errorf("refusal names %s -> %s, want %s -> %s", dotted.IssueID, dotted.DependsOnID, tt.args[0], tt.args[1]) + } + case tt.wantErr != "": + if err == nil || err.Error() != tt.wantErr { + t.Fatalf("batchDepAddEdge(%q) err = %v, want %q", tt.args, err, tt.wantErr) + } + default: + if err != nil { + t.Fatalf("batchDepAddEdge(%q): %v", tt.args, err) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("batchDepAddEdge(%q) = %+v, want %+v", tt.args, got, tt.want) + } + } + }) + } +} + func TestBatchCmd_Registered(t *testing.T) { // Ensure 'bd batch' is wired into rootCmd with the correct group. cmd, _, err := rootCmd.Find([]string{"batch"}) diff --git a/cmd/bd/dep.go b/cmd/bd/dep.go index e77e1e1c8..06d7ffe49 100644 --- a/cmd/bd/dep.go +++ b/cmd/bd/dep.go @@ -213,41 +213,6 @@ func resolveUnresolvedDepTarget(sourceID, dependsOnArg string, resolveErr error) return "", fmt.Errorf("resolving dependency ID %s: %v", dependsOnArg, resolveErr) } -// isChildOf returns true if childID is a hierarchical child of parentID. -// For example, "bd-abc.1" is a child of "bd-abc", and "bd-abc.1.2" is a child of "bd-abc.1". -func isChildOf(childID, parentID string) bool { - _, isAncestor := hierarchicalParentRelation(childID, parentID) - return isAncestor -} - -func hierarchicalParentRelation(childID, targetID string) (immediateParent string, isAncestor bool) { - // A child ID has the format "parentID.N" or "parentID.N.M" etc. - // Use ParseHierarchicalID to get the actual parent - _, actualParentID, depth := types.ParseHierarchicalID(childID) - if depth == 0 { - return "", false // Not a hierarchical ID - } - // Check if the immediate parent matches - if actualParentID == targetID { - return actualParentID, true - } - // Also check if targetID is an ancestor (e.g., "bd-abc" is an ancestor of "bd-abc.1.2") - return actualParentID, strings.HasPrefix(childID, targetID+".") -} - -// isDisallowedHierarchicalDependency reports whether an explicit dependency -// conflicts with hierarchy encoded in a dotted issue ID. The one allowed match -// is a parent-child edge to the immediate dotted-ID parent; blocking and other -// edge types to any parent/ancestor, plus parent-child edges to higher ancestors, -// remain rejected. -func isDisallowedHierarchicalDependency(fromID, toID string, depType types.DependencyType) bool { - immediateParent, isAncestor := hierarchicalParentRelation(fromID, toID) - if !isAncestor { - return false - } - return depType != types.DepParentChild || toID != immediateParent -} - var depCmd = &cobra.Command{ Use: "dep [issue-id]", GroupID: "deps", @@ -312,10 +277,6 @@ Examples: } defer toCleanup() - if isDisallowedHierarchicalDependency(fromID, toID, types.DepBlocks) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", fromID, toID) - } - opsCtx, err := issueOpsContext(ctx) if err != nil { return HandleErrorRespectJSON("%v", err) @@ -498,9 +459,8 @@ Examples: } dt := canonicalDependencyType(types.DependencyType(depType)) - if isDisallowedHierarchicalDependency(fromID, toID, dt) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", fromID, toID) - } + // The dotted-id hierarchy refusal is the role's + // (issueops.CheckDottedChildDependency), raised by AddDependencies below. if err := validateDependencyType(dt); err != nil { return HandleErrorRespectJSON("%v", err) @@ -827,7 +787,9 @@ func validateBulkDepEdges(ctx context.Context, edges []bulkDepEdge) ([]bulkDepEd current.DependsOnID = toID } - if isDisallowedHierarchicalDependency(current.IssueID, current.DependsOnID, current.Type) { + // The role refuses this edge too; asking the library rule here only lets + // the bulk report name every offending line before anything is written. + if issueops.IsDottedChildDependency(current.IssueID, current.DependsOnID, current.Type) { errs = append(errs, fmt.Sprintf("line %d: cannot add dependency: %s is already a child of %s", edge.Line, current.IssueID, current.DependsOnID)) resolved = append(resolved, current) continue diff --git a/cmd/bd/dep_proxied_server.go b/cmd/bd/dep_proxied_server.go index aee6b0306..86e10a1dd 100644 --- a/cmd/bd/dep_proxied_server.go +++ b/cmd/bd/dep_proxied_server.go @@ -177,10 +177,6 @@ func runDepBlocksProxiedServer(cmd *cobra.Command, ctx context.Context, blockerI return HandleErrorRespectJSON("%v", err) } - if isDisallowedHierarchicalDependency(blockedID, blockerID, types.DepBlocks) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", blockedID, blockerID) - } - noCycleCheck, _ := cmd.Flags().GetBool("no-cycle-check") edge := issueops.DependencyEdge{IssueID: blockedID, DependsOnID: blockerID, Type: types.DepBlocks} @@ -247,9 +243,6 @@ func runDepAddProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri toID = dependsOnArg dt := canonicalDependencyType(types.DependencyType(depType)) - if isDisallowedHierarchicalDependency(fromID, toID, dt) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", fromID, toID) - } if err := validateDependencyType(dt); err != nil { return HandleErrorRespectJSON("%v", err) @@ -299,7 +292,7 @@ func runDepAddBulkProxied(cmd *cobra.Command, ctx context.Context, file, default depEdges := make([]issueops.DependencyEdge, 0, len(edges)) for _, edge := range edges { - if isDisallowedHierarchicalDependency(edge.IssueID, edge.DependsOnID, edge.Type) { + if issueops.IsDottedChildDependency(edge.IssueID, edge.DependsOnID, edge.Type) { return HandleErrorRespectJSON("line %d: cannot add dependency: %s is already a child of %s", edge.Line, edge.IssueID, edge.DependsOnID) } if strings.HasPrefix(edge.DependsOnID, "external:") { diff --git a/cmd/bd/dep_test.go b/cmd/bd/dep_test.go index 188e1309e..1f8e23db8 100644 --- a/cmd/bd/dep_test.go +++ b/cmd/bd/dep_test.go @@ -1080,88 +1080,6 @@ func TestTreeNodeJSONIncludesEdgeFromParent(t *testing.T) { // These tests serve as regression tests ensuring the storage layer // continues to provide user-friendly error messages. -func TestIsChildOf(t *testing.T) { - tests := []struct { - name string - childID string - parentID string - want bool - }{ - // Positive cases: should be detected as child - { - name: "direct child", - childID: "bd-abc.1", - parentID: "bd-abc", - want: true, - }, - { - name: "grandchild", - childID: "bd-abc.1.2", - parentID: "bd-abc", - want: true, - }, - { - name: "nested grandchild direct parent", - childID: "bd-abc.1.2", - parentID: "bd-abc.1", - want: true, - }, - { - name: "deeply nested child", - childID: "bd-abc.1.2.3", - parentID: "bd-abc", - want: true, - }, - - // Negative cases: should NOT be detected as child - { - name: "same ID", - childID: "bd-abc", - parentID: "bd-abc", - want: false, - }, - { - name: "not a child - unrelated IDs", - childID: "bd-xyz", - parentID: "bd-abc", - want: false, - }, - { - name: "not a child - sibling", - childID: "bd-abc.2", - parentID: "bd-abc.1", - want: false, - }, - { - name: "reversed - parent is not child of child", - childID: "bd-abc", - parentID: "bd-abc.1", - want: false, - }, - { - name: "prefix but not hierarchical", - childID: "bd-abcd", - parentID: "bd-abc", - want: false, - }, - { - name: "not hierarchical ID", - childID: "bd-abc", - parentID: "bd-xyz", - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := isChildOf(tt.childID, tt.parentID) - if got != tt.want { - t.Errorf("isChildOf(%q, %q) = %v, want %v", tt.childID, tt.parentID, got, tt.want) - } - }) - } -} - // TestDepRoutedTargetOpensReadOnly is the regression guard for the dep/link // target-resolution invariant: a cross-rig dependency target is resolved by ID // only, so resolveIDWithRouting must open the routed foreign store read-only, diff --git a/cmd/bd/dependency_hierarchy_test.go b/cmd/bd/dependency_hierarchy_test.go deleted file mode 100644 index c50be2e29..000000000 --- a/cmd/bd/dependency_hierarchy_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "testing" - - "github.com/steveyegge/beads/internal/types" -) - -func TestIsDisallowedHierarchicalDependency(t *testing.T) { - tests := []struct { - name string - fromID string - toID string - depType types.DependencyType - want bool - }{ - { - name: "immediate parent-child allowed", - fromID: "bd-root.1.2", - toID: "bd-root.1", - depType: types.DepParentChild, - want: false, - }, - { - name: "immediate parent blocks rejected", - fromID: "bd-root.1.2", - toID: "bd-root.1", - depType: types.DepBlocks, - want: true, - }, - { - name: "grandparent parent-child rejected", - fromID: "bd-root.1.2", - toID: "bd-root", - depType: types.DepParentChild, - want: true, - }, - { - name: "unrelated target allowed", - fromID: "bd-root.1.2", - toID: "bd-other", - depType: types.DepBlocks, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := isDisallowedHierarchicalDependency(tt.fromID, tt.toID, tt.depType) - if got != tt.want { - t.Fatalf("isDisallowedHierarchicalDependency(%q, %q, %q) = %v, want %v", tt.fromID, tt.toID, tt.depType, got, tt.want) - } - }) - } -} diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 9a52f1415..407dae867 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -143,7 +143,7 @@ type GraphApplyDryRunRow struct { ParentID string `json:"parent_id,omitempty"` } -const graphApplyDryRunTransactionValidationNote = "dry-run validates the graph structure only; live create may still reject parent-child blocking paths after resolving stored dependencies" +const graphApplyDryRunTransactionValidationNote = "dry-run validates the graph structure only; live create may still reject parent-child blocking paths after resolving stored dependencies, or an edge that gates a dotted-id child on its own ancestor" // Known-field sets list the JSON keys recognized on each plan struct; unknown // keys warn about schema typos. Derived from json tags so they can't drift. (GH#3367) diff --git a/cmd/bd/link.go b/cmd/bd/link.go index a05be924b..2ad45f829 100644 --- a/cmd/bd/link.go +++ b/cmd/bd/link.go @@ -8,6 +8,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) var linkCmd = &cobra.Command{ @@ -63,8 +64,10 @@ Examples: defer toCleanup() dt := types.DependencyType(depType) - if isDisallowedHierarchicalDependency(fromID, toID, dt) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", fromID, toID) + // bd link writes through the store's own dependency verb rather than + // the DependencyEditor role, so it asks the role's rule directly. + if err := issueops.CheckDottedChildDependency(fromID, toID, dt); err != nil { + return HandleErrorRespectJSON("%v", err) } if !dt.IsValid() { diff --git a/cmd/bd/link_proxied_server.go b/cmd/bd/link_proxied_server.go index aef888968..c3a4c05af 100644 --- a/cmd/bd/link_proxied_server.go +++ b/cmd/bd/link_proxied_server.go @@ -10,6 +10,7 @@ import ( "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) func runLinkProxiedServer(cmd *cobra.Command, ctx context.Context, args []string) error { @@ -18,8 +19,8 @@ func runLinkProxiedServer(cmd *cobra.Command, ctx context.Context, args []string depType, _ := cmd.Flags().GetString("type") dt := types.DependencyType(depType) - if isDisallowedHierarchicalDependency(id1, id2, dt) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", id1, id2) + if err := issueops.CheckDottedChildDependency(id1, id2, dt); err != nil { + return HandleErrorRespectJSON("%v", err) } if !dt.IsValid() { diff --git a/internal/httpapi/batch_apply_test.go b/internal/httpapi/batch_apply_test.go index 824775f21..3c2c9c5c2 100644 --- a/internal/httpapi/batch_apply_test.go +++ b/internal/httpapi/batch_apply_test.go @@ -889,6 +889,19 @@ func TestApplyBatchMapsTheRolesTypedRefusalsOntoTheDocumentedCodes(t *testing.T) wantStatus: http.StatusNotFound, wantCode: "not_found", }, + { + // The dotted-id hierarchy rule is request validation (it unwraps to + // ErrValidation), never the hierarchy CONFLICT's dependency_cycle: + // it is decided from the two resolved ids, ahead of every check that + // reads the stored graph, so even a child whose parent-child edge IS + // stored gets this 400, with no hierarchy members to describe. + name: "a dotted child gated on its own parent", + err: itemErr(0, issueops.ItemDepAdd, "", "bd-1.1", + &issueops.DottedChildDependencyError{IssueID: "bd-1.1", DependsOnID: "bd-1", Type: issueops.DepBlocks}), + wantStatus: http.StatusBadRequest, + wantCode: "invalid_argument", + wantParam: "items", + }, { name: "the workspace's own validation", err: itemErr(0, issueops.ItemCreate, "", "", storage.ErrValidation), diff --git a/internal/httpapi/dependency_edit_test.go b/internal/httpapi/dependency_edit_test.go index 6761969ac..96be92261 100644 --- a/internal/httpapi/dependency_edit_test.go +++ b/internal/httpapi/dependency_edit_test.go @@ -564,6 +564,20 @@ func TestAddDependenciesMapsTheGraphsTypedRefusals(t *testing.T) { wantCode: CodeInvalidArgument, wantMembers: map[string]any{"param": "edges[0].depends_on_id"}, }, + { + // The dotted-id hierarchy rule (issueops.CheckDottedChildDependency) + // is the role's validation, so it is the 400 below and never the + // hierarchy conflict's dependency_cycle members. + name: "a dotted child gated on its own parent is a 400 on edges", + body: oneEdge, + err: &issueops.DottedChildDependencyError{ + IssueID: "bd-1.1", DependsOnID: "bd-1", Type: issueops.DepBlocks, + }, + wantStatus: http.StatusBadRequest, + wantCode: CodeInvalidArgument, + wantMembers: map[string]any{"param": "edges"}, + absentMembers: []string{"issue_id", "blocker_id", "blocker_is_ancestor"}, + }, { name: "the role's own validation refusal is a 400 on edges", body: oneEdge, diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index f62181bf3..7465f20b6 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -4277,6 +4277,9 @@ paths: or a payload its `kind` does not name, a `create` with no `title`, a duplicate `key`, a `Ref` with neither `key` nor `id` or with both, a `dep_add` with an unstorable `type` or pointing a row at ITSELF, a + `dep_add` from a dotted-id child to one of its own ancestors + (anything but the parent-child edge to its IMMEDIATE parent, judged + once the item's refs resolve and carrying `param` `items`), a `waits-for` gate outside its two values, an item a workspace's own validation refuses, and an edge endpoint that names nothing this database can see. An explicit `create.id` that already names a @@ -4396,7 +4399,9 @@ paths: and `blocker_is_ancestor`. They may come from the per-edge probe or from the END GATE, which is the one place an edge that is legal on its own and illegal in the graph this request built is caught. - Neither has a force bypass. + Neither has a force bypass. A `dep_add` from a dotted-id child to an + ancestor its own id names is neither: it is a `400` + `invalid_argument`, as on that operation. `already_exists` is a `create` item whose EXPLICIT `id` already @@ -5598,13 +5603,19 @@ paths: is empty after trimming, longer than 256 bytes or carrying control characters, an empty or over-long `edges` array, a blank or over-long endpoint id, an unstorable `type`, an edge that points an - issue at ITSELF, or an edge endpoint that names nothing this - database can see. Nothing is written in any of these cases. + issue at ITSELF, an edge from a dotted-id child to one of its own + ancestors (anything but the parent-child edge to its IMMEDIATE + parent), or an edge endpoint that names nothing this database can + see. Nothing is written in any of these cases. A self-dependency is a `400` rather than a `409` because it is request-intrinsic: it is invalid whatever the graph holds, so it is - a refusal of a VALUE and not a statement about state. An endpoint + a refusal of a VALUE and not a statement about state. The dotted-id + child's edge is a `400` for the same reason, carrying `param` + `edges`: the two ids decide it before the graph is read, so even a + child whose parent-child edge is already stored gets this `400`, + never the `409` hierarchy refusal or `dependency_exists`. An endpoint that names nothing is a `400` rather than a `404` for `POST /v0/beads/issues:batchCreate`'s reason: the refusal is about the request BODY, and there is no id in the path to have missed. @@ -5636,6 +5647,11 @@ paths: Neither has a force bypass. The recovery for both is to rethink the edge, which is why they are one status and not a retry. + + + An edge from a dotted-id child to an ancestor its own id names is + neither: it is a `400` `invalid_argument`, decided from the two ids + before the graph is read. x-bd-codes: [dependency_cycle, dependency_exists] content: application/problem+json: diff --git a/internal/httpclient/dependencyeditor.go b/internal/httpclient/dependencyeditor.go index a94a38d89..dd1c53222 100644 --- a/internal/httpclient/dependencyeditor.go +++ b/internal/httpclient/dependencyeditor.go @@ -135,5 +135,9 @@ func checkEdge(i int, edge issueops.DependencyEdge) error { case edge.IssueID == edge.DependsOnID: return fmt.Errorf("%s: %w", edge.IssueID, issueops.ErrSelfDependency) } - return nil + // The dotted-id hierarchy rule, for the self-dependency check's reason: the + // server refuses it as a 400 that arrives here as a bare ErrValidation, + // while the role promises the typed *DottedChildDependencyError. It is the + // library's own check, not a restatement. + return issueops.CheckDottedChildDependency(edge.IssueID, edge.DependsOnID, edge.Type) } diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index 1117c51b8..e7aa98872 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -265,6 +265,10 @@ func TestServedBatchApplyCloseItemsAnswerToTheCloseGuards(t *testing.T) { conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, t.Context(), newServedBatchApplyFixture(t, "hbacg")) } +func TestServedBatchApplyRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { + conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, t.Context(), newServedBatchApplyFixture(t, "hbadot")) +} + // TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed wires the dep_add // HasSpawner/ThreadID lineage conformance case onto the http leg (S5): a // served default Config always advertises CapBatchApplyDepAddLineage (it is diff --git a/internal/httpclient/served_memories_deps_test.go b/internal/httpclient/served_memories_deps_test.go index 014f38ce3..5550f0b4b 100644 --- a/internal/httpclient/served_memories_deps_test.go +++ b/internal/httpclient/served_memories_deps_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -389,6 +390,10 @@ func TestServedDependencyEditorAcceptsBlockingAcrossIssueTypes(t *testing.T) { conformance.RunDependencyEditorAcceptsBlockingAcrossIssueTypes(t, t.Context(), newServedDependencyEditorFixture(t, "hd32")) } +func TestServedDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { + conformance.RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t, t.Context(), newServedDependencyEditorFixture(t, "hddot")) +} + func TestServedDependencyEditorGateScopeFollowsTheEdgeType(t *testing.T) { conformance.RunDependencyEditorGateScopeFollowsTheEdgeType(t, t.Context(), newServedDependencyEditorFixture(t, "hd33")) } @@ -433,3 +438,37 @@ func TestServedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *test func TestServedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t *testing.T) { conformance.RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t, t.Context(), newServedDependencyEditorFixture(t, "hd41")) } + +// TestServedServerRefusesADottedChildGatedOnItsOwnParent sends the edge on the +// raw wire, past the client's own CheckDottedChildDependency, so what refuses +// it is the SERVER's role — the leg a non-bd HTTP caller (gc, curl) reaches. +// Before the rule moved into issueops only the CLI refused it, and this edge +// landed. +func TestServedServerRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { + env := newServedEnv(t, "hddotw") + parent := env.prefix + "-dotted" + child := parent + ".1" + for _, id := range []string{parent, child} { + if err := env.createIssue(t.Context(), &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + w, err := env.subject.roleWire("DependencyEditor") + if err != nil { + t.Fatalf("roleWire: %v", err) + } + _, err = w.AddDependencies(t.Context(), apigen.AddDependenciesRequest{ + Actor: "writer", + Edges: []apigen.DependencyEdge{{IssueId: child, DependsOnId: parent, Type: "blocks"}}, + }) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("raw-wire child -> parent blocks: error = %v, want the server's ErrValidation refusal", err) + } + var n int + if err := env.queryScalar(t.Context(), "SELECT COUNT(*) FROM dependencies WHERE issue_id = ?", []any{child}, &n); err != nil { + t.Fatalf("count edges: %v", err) + } + if n != 0 { + t.Fatalf("%d edge(s) from %s landed; the server must refuse a dotted child gated on its own parent", n, child) + } +} diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index 8ead3a15b..5d009ca38 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -270,6 +270,11 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) }) + t.Run("RefusesADottedChildGatedOnItsOwnParent", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) + }) } // newDoltBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/dolt/dependency_editor_contract_test.go b/internal/storage/dolt/dependency_editor_contract_test.go index f5fac1515..57167a4a4 100644 --- a/internal/storage/dolt/dependency_editor_contract_test.go +++ b/internal/storage/dolt/dependency_editor_contract_test.go @@ -259,6 +259,12 @@ func TestDependencyEditorAcceptsBlockingAcrossIssueTypes(t *testing.T) { conformance.RunDependencyEditorAcceptsBlockingAcrossIssueTypes(t, ctx, fixture) } +func TestDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { + fixture, ctx, cleanup := newDoltDependencyEditorFixture(t, "dotted") + defer cleanup() + conformance.RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) +} + // newDoltDependencyEditorFixture composes the backend's role fixture kit with // the accessor under test. Every hook but the accessor comes from the kit, so // the seeding and scalar-query plumbing stays identical to the other roles'. diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index 9280f9acb..6d330a163 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -196,6 +196,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("CloseItemsAnswerToTheCloseGuards", func(t *testing.T) { conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) }) + t.Run("RefusesADottedChildGatedOnItsOwnParent", func(t *testing.T) { + conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) + }) } // newEmbeddedBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/embeddeddolt/dependency_editor_contract_test.go b/internal/storage/embeddeddolt/dependency_editor_contract_test.go index 88335f2d7..d51e7edd2 100644 --- a/internal/storage/embeddeddolt/dependency_editor_contract_test.go +++ b/internal/storage/embeddeddolt/dependency_editor_contract_test.go @@ -261,6 +261,12 @@ func TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes(t *testing.T) { conformance.RunDependencyEditorAcceptsBlockingAcrossIssueTypes(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "xtype")) } +func TestEmbeddedDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "dotted")) +} + // newEmbeddedDependencyEditorFixture composes the backend's role fixture kit // with the accessor under test. Every hook but the accessor comes from the kit, // so the seeding and scalar-query plumbing stays identical to the other roles'. diff --git a/internal/storage/issueops/batch_apply.go b/internal/storage/issueops/batch_apply.go index 1621e772e..0459110eb 100644 --- a/internal/storage/issueops/batch_apply.go +++ b/internal/storage/issueops/batch_apply.go @@ -352,6 +352,12 @@ func (r *applyBatchRun) applyDepAdd(ctx context.Context, tx *sql.Tx, index int, // bound to. The planner catches only the syntactically identical pair. return itemErr(fmt.Errorf("%w: %s", publicops.ErrSelfDependency, source)) } + // The dotted-id hierarchy rule DependencyEditor applies, on the RESOLVED + // ids: a key can name a row whose minted id is a dotted child, so the planner + // cannot decide this before the creates ran. + if err := publicops.CheckDottedChildDependency(source, target, item.Type); err != nil { + return itemErr(err) + } // THE CROSS-PLANE REFUSAL IS ABOUT ROWS THIS REQUEST CREATED, and only // those. The two planes hold their edges in different tables, so an edge // between a durable row and a wisp the same batch minted is one the batch diff --git a/internal/storage/issueops/dependency_editor.go b/internal/storage/issueops/dependency_editor.go index ec8cb46b1..d1c7fac46 100644 --- a/internal/storage/issueops/dependency_editor.go +++ b/internal/storage/issueops/dependency_editor.go @@ -21,6 +21,12 @@ import ( // self-edge otherwise trips the cycle check and reports the wrong refusal, and // SkipPerEdgeCycleCheck would skip it entirely. // +// The dotted-id hierarchy rule (publicops.CheckDottedChildDependency) is here +// for the same reason: it is decided from the request alone, and putting it in +// the shared validator is what makes every leg — dolt, embedded, the unit of +// work and the server behind httpapi — refuse a child gated on its own dotted +// parent, not only the CLI that used to check it before calling the role. +// // The type check is that there IS a type — non-empty, within the column's // length. It is deliberately not a membership test: the vocabulary is an open, // workspace-configurable set (see the Dep* constants), so refusing an unlisted @@ -43,6 +49,9 @@ func ValidateAddDependenciesRequest(request publicops.AddDependenciesRequest) er if edge.IssueID == edge.DependsOnID { return fmt.Errorf("%w: %s cannot depend on itself", domain.ErrSelfDependency, edge.IssueID) } + if err := publicops.CheckDottedChildDependency(edge.IssueID, edge.DependsOnID, edge.Type); err != nil { + return err + } } return nil } diff --git a/internal/storage/uow/batch_applier.go b/internal/storage/uow/batch_applier.go index b26025a5c..8562f63b7 100644 --- a/internal/storage/uow/batch_applier.go +++ b/internal/storage/uow/batch_applier.go @@ -461,6 +461,9 @@ func (r *uowApplyRun) applyDepAdd(ctx context.Context, index int, item *publicop if source == target { return itemErr(fmt.Errorf("%w: %s", publicops.ErrSelfDependency, source)) } + if err := publicops.CheckDottedChildDependency(source, target, item.Type); err != nil { + return itemErr(err) + } sourceWisp, sourceMine := r.planes[source] targetWisp, targetMine := r.planes[target] if sourceMine && targetMine && sourceWisp != targetWisp { diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index 2d8b4673d..edfa1af3d 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -166,6 +166,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("CloseItemsAnswerToTheCloseGuards", func(t *testing.T) { conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) }) + t.Run("RefusesADottedChildGatedOnItsOwnParent", func(t *testing.T) { + conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) + }) } func newUOWBatchApplyFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchApplyFixture { diff --git a/internal/storage/uow/dependency_editor_contract_test.go b/internal/storage/uow/dependency_editor_contract_test.go index 855ec2521..3d2cc274a 100644 --- a/internal/storage/uow/dependency_editor_contract_test.go +++ b/internal/storage/uow/dependency_editor_contract_test.go @@ -66,6 +66,7 @@ func TestUOWDependencyEditorContract(t *testing.T) { {name: "AcceptsADiamond", run: conformance.RunDependencyEditorAcceptsADiamond}, {name: "GateScopeFollowsTheEdgeType", run: conformance.RunDependencyEditorGateScopeFollowsTheEdgeType}, {name: "AcceptsBlockingAcrossIssueTypes", run: conformance.RunDependencyEditorAcceptsBlockingAcrossIssueTypes}, + {name: "RefusesADottedChildGatedOnItsOwnParent", run: conformance.RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent}, } { t.Run(test.name, func(t *testing.T) { test.run(t, ctx, fixture) diff --git a/issueops/dependencyeditor.go b/issueops/dependencyeditor.go index f9b0683eb..16cfc1b5b 100644 --- a/issueops/dependencyeditor.go +++ b/issueops/dependencyeditor.go @@ -2,6 +2,8 @@ package issueops import ( "context" + "fmt" + "strings" "github.com/steveyegge/beads/internal/types" ) @@ -147,12 +149,20 @@ type RemoveDependencyResult struct { type DependencyEditor interface { // AddDependencies asserts every edge in the request as ONE durable act: // one transaction, one history entry. Any refused edge refuses the whole - // request and writes nothing — a self-dependency (ErrSelfDependency), a - // scheduling cycle (ErrDependencyCycle), a conflicting type on a pair that - // already has an edge (*DependencyTypeConflictError), a blocking edge that - // would gate an issue on its own ancestor or descendant - // (*DependencyHierarchyConflictError), or a SOURCE that does not exist - // (ErrDependencySourceNotFound). + // request and writes nothing — a self-dependency (ErrSelfDependency), an + // explicit edge from a dotted-id child to its own ancestor + // (*DottedChildDependencyError), a scheduling cycle (ErrDependencyCycle), + // a conflicting type on a pair that already has an edge + // (*DependencyTypeConflictError), a blocking edge that would gate an issue + // on its own ancestor or descendant (*DependencyHierarchyConflictError), + // or a SOURCE that does not exist (ErrDependencySourceNotFound). + // + // The dotted-child refusal is decided from the request alone, before any + // read, so it takes precedence over every refusal that reads stored rows: + // a child whose parent-child edge to that parent is already stored (the + // `bd create --parent` shape) is refused with *DottedChildDependencyError, + // which matches ErrValidation, not with the hierarchy or type conflict the + // stored edge would otherwise raise. // // A target's existence is checked only where the backend can see it. A // target may legitimately be an "external:" reference or an issue in @@ -188,3 +198,72 @@ type DependencyEditor interface { // missing edge is Removed false with a nil error, not ErrNotFound. RemoveDependency(ctx context.Context, req RemoveDependencyRequest) (RemoveDependencyResult, error) } + +// DottedChildDependencyError refuses an explicit edge from a dotted-id child to +// its own ancestor. The id already says "bd-abc.1 is under bd-abc": a child +// inherits its parent's completion through the hierarchy, so a blocking (or any +// other) explicit edge back up the tree either duplicates that structure or, for +// a blocking type, deadlocks it — the parent cannot close before the child, and +// the child would wait on the parent. +// +// The one edge it allows is the hierarchy itself: a parent-child edge to the +// IMMEDIATE dotted parent. A parent-child edge to a higher ancestor is refused +// too, because it would contradict the parent the id names. +// +// It unwraps to ErrValidation: the refusal is decidable from the two ids and +// the type alone, with no read, so it is a property of the request. +// +// The message is the sentence `bd dep add` already printed, kept verbatim, so +// it says "deadlock" whatever the type; Type is carried for callers that +// inspect the refusal and is not rendered. +type DottedChildDependencyError struct { + IssueID string + DependsOnID string + Type DependencyType +} + +func (e *DottedChildDependencyError) Error() string { + return fmt.Sprintf("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", + e.IssueID, e.DependsOnID) +} + +// Unwrap classifies the refusal as request validation. +func (e *DottedChildDependencyError) Unwrap() error { return ErrValidation } + +// IsDottedChildDependency reports whether an edge from issueID to dependsOnID +// of type depType is the dotted-id hierarchy conflict CheckDottedChildDependency +// refuses. It is exported for callers that phrase the refusal themselves (a +// bulk report naming the offending line); every role enforces it through +// CheckDottedChildDependency. +func IsDottedChildDependency(issueID, dependsOnID string, depType DependencyType) bool { + _, immediateParent, depth := types.ParseHierarchicalID(issueID) + if depth == 0 || dependsOnID == "" { + return false + } + if dependsOnID != immediateParent && !strings.HasPrefix(issueID, dependsOnID+".") { + return false + } + return depType != DepParentChild || dependsOnID != immediateParent +} + +// CheckDottedChildDependency is the ONE enforcement of the dotted-id hierarchy +// rule. Every DependencyEditor implementation applies it to each requested edge +// and every BatchApplier applies it to each dep_add item once its refs resolve, +// so `bd dep add`, an HTTP dependencies:add, and a batch apply (including +// `bd create --graph`'s own edges) refuse the same edge the same way. Two CLI +// paths that write an edge through a store verb instead of a role call it +// themselves: `bd link` and `bd batch`'s `dep add`. +// +// Other edge writers do not apply it yet, so they can still store an edge the +// roles refuse. The known ones are `bd relate`; the edges a create writes with +// its issue (`bd create --deps`/`--waits-for`, and Lifecycle.Create's ParentID, +// Dependencies and WaitsFor); `bd duplicate`, `bd supersede` and +// `bd duplicates --auto-merge`'s link and reparenting; and the tracker engine's +// createDependencies. The molecule, template and swarm writers have not been +// audited against it. bd-25s1wh tracks the gap. +func CheckDottedChildDependency(issueID, dependsOnID string, depType DependencyType) error { + if IsDottedChildDependency(issueID, dependsOnID, depType) { + return &DottedChildDependencyError{IssueID: issueID, DependsOnID: dependsOnID, Type: depType} + } + return nil +} diff --git a/issueops/issueops_external_test.go b/issueops/issueops_external_test.go index 4f12dffbc..fd8f81ce2 100644 --- a/issueops/issueops_external_test.go +++ b/issueops/issueops_external_test.go @@ -340,3 +340,58 @@ func (operationsProbe) Close(context.Context, issueops.CloseRequest) (issueops.C func (operationsProbe) Reopen(context.Context, issueops.ReopenRequest) (issueops.ReopenResult, error) { return issueops.ReopenResult{}, nil } + +// TestCheckDottedChildDependency pins the dotted-id hierarchy rule every +// DependencyEditor and BatchApplier enforces. It absorbs the CLI's former +// isChildOf / isDisallowedHierarchicalDependency tables. +func TestCheckDottedChildDependency(t *testing.T) { + for _, tt := range []struct { + name string + from string + to string + depType issueops.DependencyType + want bool + }{ + {"direct child blocks on parent", "bd-abc.1", "bd-abc", issueops.DepBlocks, true}, + {"grandchild blocks on grandparent", "bd-abc.1.2", "bd-abc", issueops.DepBlocks, true}, + {"grandchild blocks on immediate parent", "bd-abc.1.2", "bd-abc.1", issueops.DepBlocks, true}, + {"deeply nested child blocks on root", "bd-abc.1.2.3", "bd-abc", issueops.DepBlocks, true}, + {"child relates to parent", "bd-abc.1", "bd-abc", issueops.DepRelated, true}, + {"grandchild parent-child to grandparent", "bd-root.1.2", "bd-root", issueops.DepParentChild, true}, + {"immediate parent-child allowed", "bd-root.1.2", "bd-root.1", issueops.DepParentChild, false}, + {"same id", "bd-abc", "bd-abc", issueops.DepBlocks, false}, + {"unrelated ids", "bd-xyz", "bd-abc", issueops.DepBlocks, false}, + {"unrelated target from a dotted child", "bd-root.1.2", "bd-other", issueops.DepBlocks, false}, + {"sibling", "bd-abc.2", "bd-abc.1", issueops.DepBlocks, false}, + {"parent on its child", "bd-abc", "bd-abc.1", issueops.DepBlocks, false}, + {"prefix but not hierarchical", "bd-abcd", "bd-abc", issueops.DepBlocks, false}, + {"textual prefix of a dotted id is not an ancestor", "bd-abc.10", "bd-abc.1", issueops.DepBlocks, false}, + {"external target", "bd-abc.1", "external:bd-abc", issueops.DepBlocks, false}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := issueops.IsDottedChildDependency(tt.from, tt.to, tt.depType); got != tt.want { + t.Fatalf("IsDottedChildDependency(%q, %q, %q) = %v, want %v", tt.from, tt.to, tt.depType, got, tt.want) + } + err := issueops.CheckDottedChildDependency(tt.from, tt.to, tt.depType) + if !tt.want { + if err != nil { + t.Fatalf("CheckDottedChildDependency = %v, want nil", err) + } + return + } + var dotted *issueops.DottedChildDependencyError + if !errors.As(err, &dotted) || !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("CheckDottedChildDependency = %v, want *DottedChildDependencyError matching ErrValidation", err) + } + if dotted.IssueID != tt.from || dotted.DependsOnID != tt.to || dotted.Type != tt.depType { + t.Fatalf("refusal = %#v, want the edge it refused", dotted) + } + // The CLI prints this sentence verbatim, so it is behaviour. + want := "cannot add dependency: " + tt.from + " is already a child of " + tt.to + + ". Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock" + if err.Error() != want { + t.Fatalf("message = %q, want %q", err.Error(), want) + } + }) + } +} From c21880105fc1608a58f8472fdeff8f57a3f0c263 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 23:29:35 +0000 Subject: [PATCH 026/151] ci(bazel): split embedded conformance core into two targets (#7446) //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test ran conformance.RunAll minus Audit as one unsharded action, 108-150s on rbe-west, the embedded lane's longest after #7443. Every RunAll group builds its own store from the factory, so the groups share no state and can run in separate processes; about half the time was two groups, ReadyCountsPageChunking (~33s) and Portable (~20s). The new embeddeddolt_conformance_core_slow_test runs exactly those two groups (-test.run=^TestConformance$/^(ReadyCountsPageChunking|Portable)$), and the core target skips them along with Audit. The core target skips only named groups, so any group added to or renamed in RunAll still runs there: between them the two targets run every core group exactly once (rbe-west: 57 + 2 groups, no overlap, the same 59 as before). No test changed. The frozen-partition check now derives both targets' selectors from the retired job's core selector plus conformanceCoreSlowGroups; the retired-lane arg pins, bazel.yml's --whole list and the BUILD rules carry the new target. rbe-west, `bazel test //... --config=embedded --nocache_test_results`: conformance core 148s -> 55s + 57s. Bead: ga-vnycm2.31 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 1 + internal/storage/embeddeddolt/BUILD.bazel | 35 ++++++++++++++++-- scripts/ci_workflow_test.go | 43 ++++++++++++++++++++--- scripts/pr_risk_bazel_coverage_test.go | 8 +++-- 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 632b264d0..eeb98a410 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1399,6 +1399,7 @@ jobs: --suite //cmd/bd:bd_embedded_part2_test .github/scripts/embedded-test-shard.sh 50@50/100 --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 40 --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test + --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_slow_test --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test - name: Upload test logs diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 7997a0f13..474d6cc45 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -315,7 +315,13 @@ sh_test( ) # PR Risk's "Test (Embedded Dolt Conformance - core)" job: conformance.RunAll -# except its Audit group, with the job's binary flags but -test.timeout. +# except its Audit group, with the job's binary flags but -test.timeout. It +# runs as two targets: this one skips Audit and the groups +# embeddeddolt_conformance_core_slow_test (below) runs, so the two run every +# core group exactly once between them (a group added to RunAll lands here). +# As one action it took 108-150s on rbe-west, ~half of it those two groups. +# Every RunAll group builds its own store from the factory, so the split +# shares no state. # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, @@ -335,7 +341,32 @@ sh_test( "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$", - "-test.skip=^TestConformance$$/^Audit$$", + "-test.skip=^TestConformance$$/^(Audit|ReadyCountsPageChunking|Portable)$$", + ], + data = [":embeddeddolt_test"], + env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, + tags = ["embedded"], +) + +# The slow groups of the core conformance partition (see +# embeddeddolt_conformance_core_test, which skips exactly these): +# ReadyCountsPageChunking (~33s) and Portable (~20s) under -race. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "embeddeddolt_conformance_core_slow_test", + size = "large", + # -test.timeout=19m: as embeddeddolt_conformance_core_test. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :embeddeddolt_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + "-test.run=^TestConformance$$/^(ReadyCountsPageChunking|Portable)$$", ], data = [":embeddeddolt_test"], env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index cabb5ed54..5775326b9 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -4393,12 +4393,32 @@ func TestBazelEmbeddedJobRunsEmbeddedTier(t *testing.T) { } // The retired test-embedded-conformance job's two partitions, frozen. + // Core runs as two targets: embeddeddolt_conformance_core_test runs the + // job's core selector but also skips conformanceCoreSlowGroups, which + // embeddeddolt_conformance_core_slow_test runs instead, so between them + // they run every core group exactly once. conformance := map[string]string{ "core": `/tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$' -test.skip '^TestConformance$/^Audit$'`, "audit": `/tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$/^Audit$'`, } + slow := strings.Join(conformanceCoreSlowGroups, "|") + // What each target's rule must say instead of the job's own selector. + retarget := map[string]map[string]string{ + "embeddeddolt_conformance_core_test": { + `"-test.skip=^TestConformance$$/^Audit$$",`: `"-test.skip=^TestConformance$$/^(Audit|` + slow + `)$$",`, + }, + "embeddeddolt_conformance_core_slow_test": { + `"-test.run=^TestConformance$$",`: `"-test.run=^TestConformance$$/^(` + slow + `)$$",`, + `"-test.skip=^TestConformance$$/^Audit$$",`: "", + }, + } quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`) - for partition, target := range map[string]string{"core": "embeddeddolt_conformance_core_test", "audit": "embeddeddolt_conformance_audit_test"} { + for _, c := range []struct{ partition, target string }{ + {"core", "embeddeddolt_conformance_core_test"}, + {"core", "embeddeddolt_conformance_core_slow_test"}, + {"audit", "embeddeddolt_conformance_audit_test"}, + } { + partition, target := c.partition, c.target run := conformance[partition] fields := strings.Fields(run) if len(fields) == 0 || fields[0] != "/tmp/embeddeddolt-test" { @@ -4406,15 +4426,22 @@ func TestBazelEmbeddedJobRunsEmbeddedTier(t *testing.T) { } var want []string for _, m := range quoted.FindAllStringSubmatch(run, -1) { + var w string switch { case m[1] != "": - want = append(want, `"`+m[1]+"="+strings.ReplaceAll(m[2], "$", "$$")+`",`) + w = `"` + m[1] + "=" + strings.ReplaceAll(m[2], "$", "$$") + `",` case strings.HasPrefix(m[3], "-test.timeout="): // Documented deviation: Bazel kills at 1200s without a // goroutine dump, so the variant's Go timeout is 19m. - want = append(want, `"-test.timeout=19m",`) + w = `"-test.timeout=19m",` default: - want = append(want, `"`+m[3]+`",`) + w = `"` + m[3] + `",` + } + if r, ok := retarget[target][w]; ok { + w = r + } + if w != "" { + want = append(want, w) } } if len(want) < 4 { @@ -4429,6 +4456,14 @@ func TestBazelEmbeddedJobRunsEmbeddedTier(t *testing.T) { } } +// conformanceCoreSlowGroups are the conformance.RunAll groups +// embeddeddolt_conformance_core_slow_test runs and +// embeddeddolt_conformance_core_test skips. The core target skips only these +// names, so a group renamed or added in RunAll still runs there: a name here +// that RunAll no longer has matches nothing in either target and loses no +// group, only the split (core gets slower again). +var conformanceCoreSlowGroups = []string{"ReadyCountsPageChunking", "Portable"} + // bazel-pure replaces pr.yml's check-cmd-bd-puregeo-tests job: the same pure // cmd/bd test selector, the same pure build set, and the js/wasm hook test // under the same exact-count guard. diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 0a57f6785..c2e206425 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -344,7 +344,7 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { // The cmd/bd block is split over embeddedCmdTargets: one // --suite each, SHARDS@OFFSET/TOTAL (added below). {"test-embedded-storage", "Test", "//internal/storage/embeddeddolt:embeddeddolt_embedded_test", ".github/scripts/embedded-storage-test-shard.sh", -1}, - }, []string{"//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test", "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test"}}, + }, []string{"//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test", "//internal/storage/embeddeddolt:embeddeddolt_conformance_core_slow_test", "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test"}}, {bazelProxiedJobName, "doltserver-proxied", []suite{ // bazel-proxied runs its own duration-balanced manifest block // (scripts/ci/proxied_test_durations.json), not PR Risk's frozen @@ -874,7 +874,11 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, }, "//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test": { - []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$", "-test.skip=^TestConformance$$/^Audit$$"}, + []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$", "-test.skip=^TestConformance$$/^(Audit|ReadyCountsPageChunking|Portable)$$"}, + map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, + "//internal/storage/embeddeddolt:embeddeddolt_conformance_core_slow_test": { + []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$/^(ReadyCountsPageChunking|Portable)$$"}, map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, }, "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test": { From 1f1e82ab083b7ff6e9876d847a39a1e837d5f680 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 01:37:27 +0000 Subject: [PATCH 027/151] fix(issueops): one default priority for every create path (#7426) * fix(issueops): one create priority default; HTTP absent priority is P2 The CLI defaulted `bd create` to P2, but issues:create, issues:batchCreate and batch:apply create items left Issue.Priority at 0 when `priority` was absent, so a gc/curl create stored P0 (critical) against the spec's "absent means the default". The default now lives in one place, the shared create preparation (PreparePublicCreateRequest), behind a presence flag because Issue.Priority's zero value is P0: - issueops.CreateRequest.DefaultPriority, BatchCreateItem.DefaultPriority and CreateItem.DefaultPriority ask for issueops.DefaultCreatePriority (2); setting it with a non-zero priority is ErrValidation (ValidateCreatePriority, shared with the HTTP client); - every Lifecycle.Create, BatchCreator and BatchApplier leg (dolt, embedded, uow) reaches it; CloneCreateBatchRequest now keeps scalar item members; - the three handlers set the flag for an absent member and never make up 0; the HTTP client omits `priority` for it (wirePriority) only when the server advertises the new additive issues.create.defaultPriority token, read from the cached handshake before the dial; an older server reads an absent priority as P0, so the client sends DefaultCreatePriority (2) explicitly to it (pinCreateDefaultPriority). An explicit 0 is still sent; - `bd create` without --priority, `bd create --graph` nodes without priority and `bd create --file` templates without ### Priority rely on the flag; the flag's displayed default and the dry-run previews read the library constant. CLI output is byte-identical (checked against origin/main). Spec: the three priority descriptions now say the create default is P2 on a server advertising issues.create.defaultPriority (apigen regenerated); the wireshape golden already had priority optional, so no wire-shape change. Conformance: RunLifecycleCreateAppliesTheDefaultPriority, RunBatchCreatorAppliesTheDefaultPriority and RunBatchApplyAppliesTheDefaultPriority on dolt, embedded, uow and http, plus handler and client encoding tests, and TestCreateSendsTheDefaultPriorityToAServerThatPredatesIt (new client, handshake without the token: all three create roles send 2). Co-Authored-By: Claude Opus 5.5 * fix(issueops): apply review fixups for the default create priority (bd-kb61wg) Apply the minor and nit findings from the review of #7426: - CHANGELOG: tell out-of-tree storage backends they must honor DefaultPriority (store issueops.DefaultCreatePriority, refuse it with a non-zero priority as ErrValidation) and which conformance cases check it; scope "CLI output is unchanged" to the in-tree backends. - bd create --file: pin the default-priority path. The e2e test asserts the printed priorities, and a buildMarkdownBatchRequest unit test pins the flag and priority for an absent, a 0 and a 1 `### Priority`. Deleting `DefaultPriority: !template.PrioritySet` now fails both. - bd create --file: warn on stderr about a `### Priority` that does not parse, as the `### Type` case already does, instead of dropping it silently. - httpclient: document pinCreateDefaultPriority's same-build assumption (the handshake is cached per Store, so a stale one is silent here), and test its no-context and failed-handshake branches on all three roles. - Formula steps, `bd q` and `bd todo add` spell issueops.DefaultCreatePriority instead of the literal 2. - Conformance: the batch-creator and batch-apply default-priority cases also pin DefaultPriority with a non-zero priority as ErrValidation with nothing written. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 * fix(issueops): mark the out-of-tree DefaultPriority change BREAKING (bd-0tcn1o) Apply the minor findings from the relaunch review of #7426: - CHANGELOG: move the obligation on out-of-tree storage backends out of the `### Fixed` entry into its own `### Changed` entry, led by "BREAKING (out-of-tree storage backends)" as the close-guard change and the earlier out-of-tree contract changes are; RELEASING.md asks breaking changes to carry their migration. The entry says what such a backend used to receive from `bd create`, `--graph` and `--file` (priority 2), what it receives now (priority 0 with DefaultPriority set), the migration, and the conformance cases that check it. The `### Fixed` entry keeps the HTTP P2 fix, points at the new entry, and names the stale-handshake residual that only a code comment named: a create that lands on an older build behind the same URL still stores P0. - bd cook: the proto root, the gate issue and the cooked formula root spell issueops.DefaultCreatePriority instead of the literal 2, as the formula steps in the same file already do. No behavior change. Not taken: - The literal 2s in create_form.go, gate.go and linear.go, and the "default 2" in todo.go's --priority help: explicit P2s outside the migrated paths (the help string also feeds the generated CLI reference). Left for a follow-up sweep. - Reusing in.priorityDefault in the direct bd create route: that route builds no createInput (gatherCreateInput runs only on the proxied and --file routes, and each returns), so there is nothing to reuse. - A member-level range check in the single-create HTTP decode, as the batch shapes have: the asymmetry predates this PR, and the role still refuses an out-of-range priority on all three shapes. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 40 ++++ backend/conformance/batch_apply_contract.go | 45 ++++ backend/conformance/batch_creator_contract.go | 60 +++++ .../conformance/lifecycle_create_contract.go | 51 +++++ backend/conformance/role_bundle_cases.go | 3 + cmd/bd/cook.go | 9 +- cmd/bd/create.go | 25 +- cmd/bd/create_input.go | 2 + cmd/bd/create_markdown_batch_test.go | 7 + cmd/bd/create_proxied_server.go | 7 +- cmd/bd/create_proxied_server_test.go | 13 +- cmd/bd/graph_apply.go | 24 +- cmd/bd/graph_apply_test.go | 27 +++ cmd/bd/markdown.go | 35 ++- cmd/bd/markdown_test.go | 75 ++++++ cmd/bd/quick.go | 4 +- cmd/bd/todo.go | 3 +- internal/httpapi/BUILD.bazel | 1 + internal/httpapi/apigen/types.gen.go | 8 +- internal/httpapi/batch_apply.go | 4 +- internal/httpapi/batch_create.go | 4 +- internal/httpapi/create.go | 4 + internal/httpapi/create_priority_test.go | 99 ++++++++ internal/httpapi/routes.go | 10 +- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 27 ++- internal/httpclient/BUILD.bazel | 1 + internal/httpclient/batchapplier.go | 20 +- internal/httpclient/batchcreator.go | 20 +- internal/httpclient/bulk_roles_test.go | 4 +- internal/httpclient/create_priority_test.go | 215 ++++++++++++++++++ internal/httpclient/lifecycle.go | 74 +++++- internal/httpclient/lifecycle_create_test.go | 2 +- .../httpclient/served_batch_apply_test.go | 4 + internal/httpclient/served_bulk_test.go | 4 + .../served_lifecycle_create_test.go | 4 + internal/httpclient/wire/handshake.go | 11 +- internal/httpclient/wire/vocabulary_test.go | 4 +- internal/httpclient/write_ledger_test.go | 21 +- .../storage/dolt/batch_apply_contract_test.go | 5 + .../dolt/batch_creator_contract_test.go | 3 + .../dolt/lifecycle_create_contract_test.go | 3 + .../embeddeddolt/batch_apply_contract_test.go | 3 + .../batch_creator_contract_test.go | 3 + .../lifecycle_create_contract_test.go | 3 + internal/storage/issueops/batch_apply.go | 7 +- internal/storage/issueops/create_batch.go | 18 +- internal/storage/issueops/public_create.go | 25 ++ internal/storage/uow/batch_applier.go | 7 +- .../storage/uow/batch_apply_contract_test.go | 3 + .../uow/batch_creator_contract_test.go | 3 + .../uow/lifecycle_create_contract_test.go | 3 + issueops/batchapplier.go | 3 + issueops/batchcreator.go | 2 + issueops/issueops.go | 15 ++ 55 files changed, 1000 insertions(+), 79 deletions(-) create mode 100644 internal/httpapi/create_priority_test.go create mode 100644 internal/httpclient/create_priority_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5477be6ae..6b52a5b5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -230,6 +230,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 pinned issues or beads another actor holds must send `force` (`Force` on the request), or reclaim the bead first; the close operations never close a template, forced or not (pour it instead). +- **BREAKING (out-of-tree storage backends): a create without a priority now + arrives as priority 0 with `DefaultPriority` set.** `bd create` without + `--priority`, a `bd create --graph` node without `priority` and a + `bd create --file` template without a valid `### Priority` used to reach a + backend's `IssueLifecycle().Create`, `BatchCreator().CreateBatch` and + `BatchApplier().ApplyBatch` as priority 2. They now arrive as priority 0 + with `issueops.CreateRequest.DefaultPriority` (or `BatchCreateItem` / + `CreateItem.DefaultPriority`) set, as does an HTTP create without + `priority` when `bd serve` fronts the backend. The fields are additive, so + such a backend compiles unchanged, but the shared preparation that applies + the default lives under `internal/`, so a backend that ignores the flag now + stores P0 (critical) for those `bd create` paths, and still stores P0 for + that HTTP create. Migration: store `issueops.DefaultCreatePriority` when + `DefaultPriority` is set, and refuse `DefaultPriority` with a non-zero + priority as `ErrValidation`; `conformance.RunRoleContracts` checks both + through `RunLifecycleCreateAppliesTheDefaultPriority`, + `RunBatchCreatorAppliesTheDefaultPriority` and + `RunBatchApplyAppliesTheDefaultPriority`. ### Fixed - **An update of a template is now refused for every caller, not only @@ -279,6 +297,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 stored (a `bd create --parent` child) an HTTP caller now gets that 400 where it used to get 409 `dependency_cycle` (a blocking type) or `dependency_exists` (any other type). +- An HTTP create (`issues:create`, `issues:batchCreate`, a `batch:apply` + create item) that omits `priority` now stores the create default, P2, as the + spec documents, instead of P0 (critical). The default lives in one place: + `issueops.CreateRequest.DefaultPriority` (and `BatchCreateItem` / + `CreateItem.DefaultPriority`) asks the shared create preparation to store + `issueops.DefaultCreatePriority`; the handlers set it for an absent member, + the HTTP client omits `priority` for it only when the server advertises the + new additive `issues.create.defaultPriority` handshake token (an older + `bd serve` reads an absent `priority` as P0, so a new client sends `2` + explicitly to it), and `bd create` (no `--priority`), + `bd create --graph` (a node without `priority`) and `bd create --file` (a + template without `### Priority`) rely on it rather than spelling 2. An + explicit `0` is still P0, and `DefaultPriority` with a non-zero priority is + `ErrValidation`. CLI output is unchanged on the in-tree backends, except + that `bd create --file` now warns on stderr about a `### Priority` it cannot + parse, as it already did for `### Type`, rather than dropping it silently. + The client decides from the handshake its store cached (once per command + in `bd`), so a create that lands on an older build behind the same URL (a + `bd serve` rolled back under a live store, or mixed builds during a + rollout) still stores P0, with no error on either side. + Out-of-tree storage backends must now honor `DefaultPriority`; see the + BREAKING (out-of-tree storage backends) entry under `### Changed`. - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index f6480e47a..6c173022a 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -2734,3 +2734,48 @@ func RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t *testing.T, ctx conte } assertBatchApplyEdgeCount(t, ctx, fixture, child, parent, 1) } + +// RunBatchApplyAppliesTheDefaultPriority pins CreateItem.DefaultPriority: a +// create item that asks for the default stores publicops.DefaultCreatePriority, +// and one naming priority 0 stores P0. `bd create --graph` sends a node without +// a priority this way, and so does an HTTP batch apply create item without one. +// A create item asking for the default while naming a priority is +// ErrValidation and writes nothing, as it is for Lifecycle.Create. +func RunBatchApplyAppliesTheDefaultPriority(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + unset := batchApplyMintedIssue("defaulted") + unset.Priority = 0 + zero := batchApplyMintedIssue("critical") + zero.Priority = 0 + result := batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "d", Issue: unset, DefaultPriority: true}}, + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "z", Issue: zero}}, + }, + }) + for key, want := range map[string]int{"d": publicops.DefaultCreatePriority, "z": 0} { + id := result.Keys[key] + if id == "" { + t.Fatalf("key %q bound no id: %v", key, result.Keys) + } + if got := batchApplyCount(t, ctx, fixture, "SELECT priority FROM issues WHERE id = ?", []any{id}); got != want { + t.Errorf("stored priority of %s (%s) = %d, want %d", key, id, got, want) + } + } + + conflict := fixture.IssuePrefix + "-prio-conflict" + clash := batchApplyIssue(conflict, conflict) + clash.Priority = 3 + _, err := fixture.BatchApplier.ApplyBatch(ctx, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + ForceIDPrefix: true, + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "c", Issue: clash, DefaultPriority: true}}, + }, + }) + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("DefaultPriority with priority 3: err = %v, want ErrValidation", err) + } + assertBatchApplyRowCount(t, ctx, fixture, "issues", conflict, 0) +} diff --git a/backend/conformance/batch_creator_contract.go b/backend/conformance/batch_creator_contract.go index 0b48334f0..91353d348 100644 --- a/backend/conformance/batch_creator_contract.go +++ b/backend/conformance/batch_creator_contract.go @@ -689,3 +689,63 @@ func batchCreatorHistoryCount(t *testing.T, ctx context.Context, fixture BatchCr } return entries } + +// RunBatchCreatorAppliesTheDefaultPriority pins BatchCreateItem.DefaultPriority: +// an item that asks for the default stores publicops.DefaultCreatePriority, and +// an item naming priority 0 stores P0 in the same batch. The default is +// Lifecycle.Create's, applied in the shared preparation, so an HTTP +// batchCreate without `priority` and a `bd create --file` template without one +// land the same row. An item asking for the default while naming a priority is +// ErrValidation and writes nothing, as it is for Lifecycle.Create. +func RunBatchCreatorAppliesTheDefaultPriority(t *testing.T, ctx context.Context, fixture BatchCreatorFixture) { + t.Helper() + // Minted ids and an unset status, so the case asks nothing of the items + // but their priority — the http wire's batch item has no id or status slot. + unset := &types.Issue{Title: fixture.IssuePrefix + " defaulted", IssueType: types.TypeTask} + zero := &types.Issue{Title: fixture.IssuePrefix + " critical", IssueType: types.TypeTask} + + result, err := fixture.BatchCreator.CreateBatch(ctx, publicops.CreateBatchRequest{ + Actor: "batch-writer", + Items: []publicops.BatchCreateItem{ + {Issue: unset, DefaultPriority: true}, + {Issue: zero}, + }, + }) + if err != nil { + t.Fatalf("CreateBatch: %v", err) + } + if len(result.Issues) != 2 || result.Issues[0] == nil || result.Issues[1] == nil { + t.Fatalf("result = %v, want two created issues", result.Issues) + } + for i, want := range []int{publicops.DefaultCreatePriority, 0} { + created := result.Issues[i] + if created.Priority != want { + t.Errorf("item %d result priority = %d, want %d", i, created.Priority, want) + } + var got int + if err := fixture.QueryScalar(ctx, "SELECT priority FROM issues WHERE id = ?", []any{created.ID}, &got); err != nil { + t.Fatalf("read priority of %s: %v", created.ID, err) + } + if got != want { + t.Errorf("item %d stored priority = %d, want %d", i, got, want) + } + } + + conflict := fixture.IssuePrefix + " conflicting" + _, err = fixture.BatchCreator.CreateBatch(ctx, publicops.CreateBatchRequest{ + Actor: "batch-writer", + Items: []publicops.BatchCreateItem{ + {Issue: &types.Issue{Title: conflict, Priority: 3, IssueType: types.TypeTask}, DefaultPriority: true}, + }, + }) + if !errors.Is(err, storage.ErrValidation) { + t.Fatalf("DefaultPriority with priority 3: err = %v, want ErrValidation", err) + } + var stored int + if err := fixture.QueryScalar(ctx, "SELECT COUNT(*) FROM issues WHERE title = ?", []any{conflict}, &stored); err != nil { + t.Fatalf("count issues titled %q: %v", conflict, err) + } + if stored != 0 { + t.Errorf("issues titled %q after the conflicting priority batch = %d, want 0", conflict, stored) + } +} diff --git a/backend/conformance/lifecycle_create_contract.go b/backend/conformance/lifecycle_create_contract.go index ca906424a..ae1efb1b1 100644 --- a/backend/conformance/lifecycle_create_contract.go +++ b/backend/conformance/lifecycle_create_contract.go @@ -804,3 +804,54 @@ func lifecycleCreatePlaneName(ephemeral bool) string { } return "durable" } + +// RunLifecycleCreateAppliesTheDefaultPriority pins +// CreateRequest.DefaultPriority: a create that asks for the default stores +// publicops.DefaultCreatePriority, and one that names priority 0 stores P0 — +// the zero value is a real priority, never "absent". Asking for the default +// while naming a priority is ErrValidation and writes nothing. +// +// It is the one place the default lives: `bd create` without --priority and an +// HTTP create without `priority` both reach it through this flag, so a backend +// that dropped it would store P0 (critical) for every such create. +func RunLifecycleCreateAppliesTheDefaultPriority(t *testing.T, ctx context.Context, fixture LifecycleCreateFixture) { + t.Helper() + defaulted := fixture.IssuePrefix + "-prio-default" + critical := fixture.IssuePrefix + "-prio-zero" + conflict := fixture.IssuePrefix + "-prio-conflict" + + for _, tc := range []struct { + id string + priority int + useDefault bool + want int + }{ + {id: defaulted, useDefault: true, want: publicops.DefaultCreatePriority}, + {id: critical, priority: 0, want: 0}, + } { + created, err := fixture.Lifecycle.Create(ctx, publicops.CreateRequest{ + Actor: "writer", + DefaultPriority: tc.useDefault, + Issue: &types.Issue{ID: tc.id, Title: tc.id, Priority: tc.priority, IssueType: types.TypeTask}, + }) + if err != nil { + t.Fatalf("create %s: %v", tc.id, err) + } + if created.Issue.Priority != tc.want { + t.Errorf("create %s result priority = %d, want %d", tc.id, created.Issue.Priority, tc.want) + } + if got := lifecycleCreateRow(t, ctx, fixture, tc.id).Priority; got != tc.want { + t.Errorf("stored %s priority = %d, want %d", tc.id, got, tc.want) + } + } + + _, err := fixture.Lifecycle.Create(ctx, publicops.CreateRequest{ + Actor: "writer", + DefaultPriority: true, + Issue: &types.Issue{ID: conflict, Title: conflict, Priority: 3, IssueType: types.TypeTask}, + }) + if !errors.Is(err, storage.ErrValidation) { + t.Fatalf("DefaultPriority with priority 3: err = %v, want ErrValidation", err) + } + assertLifecycleCreateAbsent(t, ctx, fixture, conflict, "after the conflicting priority create") +} diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 0555b19a5..782a20306 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -62,6 +62,7 @@ var roleContractCases = []roleContract{ RunBatchApplySplicesTheMetadataOfATemplateItCreates, RunBatchApplyCloseItemsAnswerToTheCloseGuards, RunBatchApplyRefusesADottedChildGatedOnItsOwnParent, + RunBatchApplyAppliesTheDefaultPriority, ), roleCases("BatchCloser", "BatchCloser()", oncePerRole, @@ -104,6 +105,7 @@ var roleContractCases = []roleContract{ RunBatchCreatorRecordsNoHistoryForAnEphemeralBatch, RunBatchCreatorDoesNotMutateTheCallerRequest, RunBatchCreatorEchoesSubSecondTimestamps, + RunBatchCreatorAppliesTheDefaultPriority, ), roleCases("BlockingAnnotator", "BlockingAnnotator()", oncePerRole, @@ -459,6 +461,7 @@ var roleContractCases = []roleContract{ RunLifecycleCreateInheritsParentLabels, RunLifecycleCreateWritesEveryScalarField, RunLifecycleCreateEchoesSubSecondTimestamps, + RunLifecycleCreateAppliesTheDefaultPriority, ), roleCases("LifecycleUpdate", "IssueLifecycle()", oncePerRole, diff --git a/cmd/bd/cook.go b/cmd/bd/cook.go index 6e94309b0..853b90a67 100644 --- a/cmd/bd/cook.go +++ b/cmd/bd/cook.go @@ -14,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) // stepTypeToIssueType converts a formula step type string to a types.IssueType. @@ -454,7 +455,7 @@ func cookFormulaToSubgraph(f *formula.Formula, protoID string) (*TemplateSubgrap Title: rootTitle, Description: rootDesc, Status: types.StatusOpen, - Priority: 2, + Priority: issueops.DefaultCreatePriority, IssueType: types.TypeMolecule, IsTemplate: true, CreatedAt: time.Now(), @@ -512,7 +513,7 @@ func createGateIssue(step *formula.Step, parentID string) *types.Issue { Title: title, Description: fmt.Sprintf("Async gate for step %s", step.ID), Status: types.StatusOpen, - Priority: 2, + Priority: issueops.DefaultCreatePriority, IssueType: "gate", AwaitType: step.Gate.Type, AwaitID: awaitID, @@ -568,7 +569,7 @@ func processStepToIssue(step *formula.Step, parentID string) *types.Issue { } // Determine priority - priority := 2 + priority := issueops.DefaultCreatePriority if step.Priority != nil { priority = *step.Priority } @@ -969,7 +970,7 @@ func cookFormula(ctx context.Context, s storage.DoltStorage, f *formula.Formula, Title: rootTitle, Description: rootDesc, Status: types.StatusOpen, - Priority: 2, + Priority: issueops.DefaultCreatePriority, IssueType: types.TypeMolecule, IsTemplate: true, CreatedAt: time.Now(), diff --git a/cmd/bd/create.go b/cmd/bd/create.go index 38692d287..4c2004882 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "time" @@ -165,6 +166,10 @@ var createCmd = &cobra.Command{ if err != nil { return HandleError("%v", err) } + // No --priority asks the role for its default (CreateRequest. + // DefaultPriority). The flag's displayed default is that same library + // constant, so the dry-run preview below still shows what will land. + priorityDefault := !cmd.Flags().Changed("priority") issueType, _ := cmd.Flags().GetString("type") assignee, _ := cmd.Flags().GetString("assignee") @@ -567,14 +572,18 @@ var createCmd = &cobra.Command{ // Label inheritance stays CLI-side (mergeCreateLabels above) because the // dry-run preview needs it too; asking the facade to inherit as well // would append the parent's labels a second time. + if priorityDefault { + issue.Priority = 0 + } result, err := ops.Create(opsCtx, issueops.CreateRequest{ - Actor: currentActor(), - Issue: issue, - ParentID: parentID, - Dependencies: createDependencyRequests(depSpecs), - WaitsFor: waitsForRequest(waitsForSpec), - ForceIDPrefix: forceCreate, - IDPrefix: createIDPrefixOverride(), + Actor: currentActor(), + Issue: issue, + ParentID: parentID, + Dependencies: createDependencyRequests(depSpecs), + WaitsFor: waitsForRequest(waitsForSpec), + ForceIDPrefix: forceCreate, + IDPrefix: createIDPrefixOverride(), + DefaultPriority: priorityDefault, }) if err != nil { // RULING R1: an occupied --id is a refusal, not a silent full-row @@ -910,7 +919,7 @@ func init() { createCmd.Flags().String("title", "", "Issue title (alternative to positional argument)") createCmd.Flags().Bool("silent", false, "Output only the issue ID (for scripting)") createCmd.Flags().Bool("dry-run", false, "Preview what would be created without actually creating") - registerPriorityFlag(createCmd, "2") + registerPriorityFlag(createCmd, strconv.Itoa(issueops.DefaultCreatePriority)) createCmd.Flags().StringP("type", "t", "task", "Issue type (bug|feature|task|epic|chore|decision|spike|story|milestone); custom types require types.custom config; aliases: enhancement/feat→feature, dec/adr→decision") createCmd.Flags().StringP("status", "s", "", "Initial status") registerCommonIssueFlags(createCmd) diff --git a/cmd/bd/create_input.go b/cmd/bd/create_input.go index f763c0e18..a0f584d8e 100644 --- a/cmd/bd/create_input.go +++ b/cmd/bd/create_input.go @@ -26,6 +26,7 @@ type createInput struct { issueType string status string priority int + priorityDefault bool // no --priority: the role's default applies assignee string externalRef string specID string @@ -180,6 +181,7 @@ func gatherCreateInput(cmd *cobra.Command, args []string) (createInput, error) { return in, HandleError("%v", err) } in.priority = priority + in.priorityDefault = !cmd.Flags().Changed("priority") in.issueType, _ = cmd.Flags().GetString("type") diff --git a/cmd/bd/create_markdown_batch_test.go b/cmd/bd/create_markdown_batch_test.go index 79c6a01f7..846aa0834 100644 --- a/cmd/bd/create_markdown_batch_test.go +++ b/cmd/bd/create_markdown_batch_test.go @@ -51,6 +51,13 @@ Second body. if !strings.Contains(out, "Created 2 issues from") { t.Errorf("create --file output = %q, want the two-issue summary", out) } + // The second template names no priority, so it asks the role for the + // default rather than spelling 2; a role that missed the ask stores P0. + for _, want := range []string{"First from file [P1, bug]", "Second from file [P2, task]"} { + if !strings.Contains(out, want) { + t.Errorf("create --file output = %q, want %q", out, want) + } + } titles := createDepsIssueTitles(t, bd, dir) for _, title := range []string{"First from file", "Second from file"} { if !titles[title] { diff --git a/cmd/bd/create_proxied_server.go b/cmd/bd/create_proxied_server.go index 4e29fcca6..f56e81cda 100644 --- a/cmd/bd/create_proxied_server.go +++ b/cmd/bd/create_proxied_server.go @@ -104,6 +104,10 @@ func runCreateProxiedSingle(_ *cobra.Command, ctx context.Context, in createInpu } issue := buildCreateIssueFromInput(in) + if in.priorityDefault { + // No --priority: the role applies its default (CreateRequest.DefaultPriority). + issue.Priority = 0 + } // Labels ride on the issue because that is where the contract reads them: // CreateRequest.Issue documents them as authoritative. issue.Labels = append([]string(nil), in.labels...) @@ -127,7 +131,8 @@ func runCreateProxiedSingle(_ *cobra.Command, ctx context.Context, in createInpu // and only this side can see it. Without this the proxied route mints // ids the workspace's own configuration forbids and the direct route // refuses. - IDPrefix: createIDPrefixOverride(), + IDPrefix: createIDPrefixOverride(), + DefaultPriority: in.priorityDefault, }) if err != nil { // RULING R1, reported the same way the direct route reports it: an diff --git a/cmd/bd/create_proxied_server_test.go b/cmd/bd/create_proxied_server_test.go index ed8e1a80f..13d8b2895 100644 --- a/cmd/bd/create_proxied_server_test.go +++ b/cmd/bd/create_proxied_server_test.go @@ -133,12 +133,19 @@ func nodeIssueFromInput(t *testing.T, node GraphApplyNode, in createInput) *type func TestGraphApplyNodeIssue_DefaultsAndOpts(t *testing.T) { t.Run("type and priority defaults", func(t *testing.T) { - issue := nodeIssueFromInput(t, GraphApplyNode{Key: "n", Title: "N"}, createInput{createdBy: "t"}) + node := GraphApplyNode{Key: "n", Title: "N"} + issue := nodeIssueFromInput(t, node, createInput{createdBy: "t"}) if issue.IssueType != types.TypeTask { t.Errorf("type default = %q, want task", issue.IssueType) } - if issue.Priority != 2 { - t.Errorf("priority default = %d, want 2", issue.Priority) + // A node without a priority leaves it to the role's default + // (CreateItem.DefaultPriority), so the materialized issue carries none; + // the dry-run preview shows the library default the apply will store. + if issue.Priority != 0 { + t.Errorf("materialized priority = %d, want 0 (the role applies the default)", issue.Priority) + } + if got := graphApplyPreviewPriority(node, issue); got != issueops.DefaultCreatePriority { + t.Errorf("preview priority = %d, want the library default %d", got, issueops.DefaultCreatePriority) } if issue.Status != types.StatusOpen { t.Errorf("status = %q, want open", issue.Status) diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 407dae867..7fe321c17 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -417,7 +417,7 @@ func emitGraphApplyDryRun(plan *GraphApplyPlan, opts GraphApplyOptions) error { Title: node.Title, Type: string(issue.IssueType), Status: status, - Priority: issue.Priority, + Priority: graphApplyPreviewPriority(node, issue), ParentKey: effectiveParentKey, ParentID: node.ParentID, }) @@ -798,7 +798,9 @@ func graphApplyNodeIssue(node GraphApplyNode, opts GraphApplyOptions, createdBy, metadataJSON = raw } - priority := 2 + // A node with no priority asks for the role's default + // (CreateItem.DefaultPriority); the number is not spelled here. + priority := 0 if node.Priority != nil { priority = *node.Priority } @@ -1019,9 +1021,10 @@ func buildGraphApplyBatchRequest(plan *GraphApplyPlan, opts GraphApplyOptions, a items = append(items, issueops.ApplyItem{ Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ - Key: node.Key, - Issue: issue, - MetadataRefs: metadataRefs, + Key: node.Key, + Issue: issue, + MetadataRefs: metadataRefs, + DefaultPriority: node.Priority == nil, }, }) } @@ -1212,3 +1215,14 @@ func graphApplySortedKeys(keys map[string]bool) []string { sort.Strings(out) return out } + +// graphApplyPreviewPriority is the priority the dry-run preview shows: the +// node's own, or — for a node that names none and so asks the role for its +// default — the library's DefaultCreatePriority, which is what the live apply +// will store. +func graphApplyPreviewPriority(node GraphApplyNode, issue *types.Issue) int { + if node.Priority == nil { + return issueops.DefaultCreatePriority + } + return issue.Priority +} diff --git a/cmd/bd/graph_apply_test.go b/cmd/bd/graph_apply_test.go index 1d8ded671..9499281fc 100644 --- a/cmd/bd/graph_apply_test.go +++ b/cmd/bd/graph_apply_test.go @@ -1616,3 +1616,30 @@ func TestBuildGraphApplyBatchRequestFieldsSurvive(t *testing.T) { } }) } + +// TestBuildGraphApplyBatchRequestLeavesAnAbsentPriorityToTheRole: a node with +// no priority becomes a create item asking for the role's default +// (CreateItem.DefaultPriority) with a zero Issue.Priority, and a node naming 0 +// stays P0 with the flag off. `bd create --graph` no longer spells the default. +func TestBuildGraphApplyBatchRequestLeavesAnAbsentPriorityToTheRole(t *testing.T) { + zero := 0 + req, err := buildGraphApplyBatchRequest(&GraphApplyPlan{Nodes: []GraphApplyNode{ + {Key: "a", Title: "absent"}, + {Key: "z", Title: "zero", Priority: &zero}, + }}, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + creates := map[string]*issueops.CreateItem{} + for _, item := range req.Items { + if item.Kind == issueops.ItemCreate { + creates[item.Create.Key] = item.Create + } + } + if a := creates["a"]; a == nil || !a.DefaultPriority || a.Issue.Priority != 0 { + t.Errorf("absent priority item = %+v, want DefaultPriority and a zero Issue.Priority", a) + } + if z := creates["z"]; z == nil || z.DefaultPriority || z.Issue.Priority != 0 { + t.Errorf("explicit 0 item = %+v, want P0 without DefaultPriority", z) + } +} diff --git a/cmd/bd/markdown.go b/cmd/bd/markdown.go index b902ad7a4..1010743c3 100644 --- a/cmd/bd/markdown.go +++ b/cmd/bd/markdown.go @@ -34,10 +34,14 @@ type IssueTemplate struct { Design string AcceptanceCriteria string Priority int - IssueType types.IssueType - Assignee string - Labels []string - Dependencies []string + // PrioritySet records that the file named a priority. Without one the + // create asks the role for its default (BatchCreateItem.DefaultPriority); + // Priority then only carries that default for display. + PrioritySet bool + IssueType types.IssueType + Assignee string + Labels []string + Dependencies []string } // parseStringList extracts a list of strings from content, splitting by comma or whitespace. @@ -77,6 +81,10 @@ func processIssueSection(issue *IssueTemplate, section, content string) { case "priority": if p := validation.ParsePriority(content); p != -1 { issue.Priority = p + issue.PrioritySet = true + } else { + fmt.Fprintf(os.Stderr, "Warning: invalid priority '%s' in '%s', using default P%d\n", + content, issue.Title, issueops.DefaultCreatePriority) } case "type": t, err := validation.ParseIssueType(content) @@ -193,8 +201,8 @@ func (s *markdownParseState) handleH2Header(matches []string) { // Start new issue s.currentIssue = &IssueTemplate{ Title: strings.TrimSpace(matches[1]), - Priority: 2, // Default priority - IssueType: "task", // Default type + Priority: issueops.DefaultCreatePriority, // the role's default, for display; see PrioritySet + IssueType: "task", // Default type } s.currentSection = "" } @@ -394,7 +402,7 @@ func buildMarkdownBatchRequest(templates []*IssueTemplate, in createInput) (issu // wire still refuses this request's Provenance and Owner (see // internal/httpclient/batchcreator.go), so `--file` is not // servable there yet. - Priority: template.Priority, + Priority: markdownItemPriority(template), IssueType: template.IssueType, Assignee: template.Assignee, Labels: template.Labels, @@ -405,7 +413,8 @@ func buildMarkdownBatchRequest(templates []*IssueTemplate, in createInput) (issu CreatedBy: in.createdBy, Owner: in.owner, }, - Dependencies: dependencies, + Dependencies: dependencies, + DefaultPriority: !template.PrioritySet, }) } return issueops.CreateBatchRequest{ @@ -497,3 +506,13 @@ func reportMarkdownBatch(issues []*types.Issue, in createInput) error { } return nil } + +// markdownItemPriority is the priority a template's create item carries: its +// own when the file named one, otherwise zero, because the item then asks the +// role for its default (BatchCreateItem.DefaultPriority). +func markdownItemPriority(template *IssueTemplate) int { + if template.PrioritySet { + return template.Priority + } + return 0 +} diff --git a/cmd/bd/markdown_test.go b/cmd/bd/markdown_test.go index e91ef89d7..c1ced444c 100644 --- a/cmd/bd/markdown_test.go +++ b/cmd/bd/markdown_test.go @@ -1,9 +1,13 @@ package main import ( + "fmt" "os" "path/filepath" + "strings" "testing" + + "github.com/steveyegge/beads/issueops" ) func TestParseMarkdownFile(t *testing.T) { @@ -254,6 +258,77 @@ func TestParseMarkdownFile_FileNotFound(t *testing.T) { } } +// An unparseable `### Priority` leaves the template to the role's default, and +// says so on stderr the way an unparseable `### Type` does. +func TestParseMarkdownFileWarnsOnAnInvalidPriority(t *testing.T) { + path := filepath.Join(t.TempDir(), "plan.md") + if err := os.WriteFile(path, []byte("## Urgent-ish\n\n### Priority\nurgent\n"), 0o600); err != nil { + t.Fatalf("write plan: %v", err) + } + var templates []*IssueTemplate + stderr := captureStderr(t, func() { + var err error + if templates, err = parseMarkdownFile(path); err != nil { + t.Errorf("parseMarkdownFile: %v", err) + } + }) + want := fmt.Sprintf("Warning: invalid priority 'urgent' in 'Urgent-ish', using default P%d", issueops.DefaultCreatePriority) + if !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want %q", stderr, want) + } + if len(templates) != 1 || templates[0].PrioritySet { + t.Fatalf("templates = %+v, want one template that leaves its priority to the role", templates) + } +} + +// A template without `### Priority` asks the role for its default instead of +// spelling 2; one that names a priority, 0 included, sends it without the ask. +func TestBuildMarkdownBatchRequestLeavesAnAbsentPriorityToTheRole(t *testing.T) { + path := filepath.Join(t.TempDir(), "plan.md") + plan := `## Names no priority + +## Names priority zero + +### Priority +0 + +## Names priority one + +### Priority +1 +` + if err := os.WriteFile(path, []byte(plan), 0o600); err != nil { + t.Fatalf("write plan: %v", err) + } + templates, err := parseMarkdownFile(path) + if err != nil { + t.Fatalf("parseMarkdownFile: %v", err) + } + request, err := buildMarkdownBatchRequest(templates, createInput{markdownFile: path}) + if err != nil { + t.Fatalf("buildMarkdownBatchRequest: %v", err) + } + want := []struct { + title string + priority int + defaultPriority bool + }{ + {"Names no priority", 0, true}, + {"Names priority zero", 0, false}, + {"Names priority one", 1, false}, + } + if len(request.Items) != len(want) { + t.Fatalf("request has %d items, want %d", len(request.Items), len(want)) + } + for i, w := range want { + item := request.Items[i] + if item.Issue.Title != w.title || item.Issue.Priority != w.priority || item.DefaultPriority != w.defaultPriority { + t.Errorf("item %d = {%q, priority %d, DefaultPriority %v}, want {%q, priority %d, DefaultPriority %v}", + i, item.Issue.Title, item.Issue.Priority, item.DefaultPriority, w.title, w.priority, w.defaultPriority) + } + } +} + func stringSlicesEqual(a, b []string) bool { if len(a) != len(b) { return false diff --git a/cmd/bd/quick.go b/cmd/bd/quick.go index e9ab99c03..3d79d3448 100644 --- a/cmd/bd/quick.go +++ b/cmd/bd/quick.go @@ -3,6 +3,7 @@ package main import ( "errors" "fmt" + "strconv" "strings" "github.com/spf13/cobra" @@ -11,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/utils" "github.com/steveyegge/beads/internal/validation" + "github.com/steveyegge/beads/issueops" ) var quickCmd = &cobra.Command{ @@ -106,7 +108,7 @@ Example: } func init() { - quickCmd.Flags().StringP("priority", "p", "2", "Priority (0-4 or P0-P4)") + quickCmd.Flags().StringP("priority", "p", strconv.Itoa(issueops.DefaultCreatePriority), "Priority (0-4 or P0-P4)") quickCmd.Flags().StringP("type", "t", "task", "Issue type") quickCmd.Flags().StringSliceP("labels", "l", []string{}, "Labels") quickCmd.Flags().String("parent", "", "Parent issue ID for hierarchical child (e.g., 'bd-a3f8e9')") diff --git a/cmd/bd/todo.go b/cmd/bd/todo.go index b05c2fc99..9ba244a3f 100644 --- a/cmd/bd/todo.go +++ b/cmd/bd/todo.go @@ -12,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" + "github.com/steveyegge/beads/issueops" ) var todoCmd = &cobra.Command{ @@ -254,7 +255,7 @@ func init() { todoCmd.AddCommand(doneTodoCmd) // Add flags - addTodoCmd.Flags().IntP("priority", "p", 2, "Priority (0-4, default 2)") + addTodoCmd.Flags().IntP("priority", "p", issueops.DefaultCreatePriority, "Priority (0-4, default 2)") addTodoCmd.Flags().StringP("description", "d", "", "Description") listTodosCmd.Flags().Bool("all", false, "Show all TODOs including completed") diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 2cc93e90c..299076f2a 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -77,6 +77,7 @@ go_test( "comments_test.go", "context_test.go", "count_test.go", + "create_priority_test.go", "create_test.go", "cycles_test.go", "delete_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 1d12b9fa7..9516738e8 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -475,7 +475,7 @@ type ApplyCreateItem struct { // Owner The human owner, which is a different member from `assignee`: the assignee is who is working it now, the owner is who it is attributed to. Owner *string `json:"owner,omitempty"` - // Priority 0 is P0/critical. Absent means the workspace default. + // Priority 0 is P0/critical. Absent means the create default, P2 (`2`), on a server advertising `issues.create.defaultPriority`; such a server never reads an absent member as `0`. An older server stores P0 for an absent member, so a client that does not see the token sends `2` explicitly. Priority *int `json:"priority,omitempty"` // Sender Who sent this, for the message-shaped rows a plan creates. Stored verbatim and interpreted by nothing on this surface. @@ -807,7 +807,7 @@ type BatchCreateItem struct { IssueType *string `json:"issue_type,omitempty"` Labels *[]string `json:"labels,omitempty"` - // Priority 0 is P0/critical. Absent means the workspace default. + // Priority 0 is P0/critical. Absent means the create default, P2 (`2`), on a server advertising `issues.create.defaultPriority`; such a server never reads an absent member as `0`. An older server stores P0 for an absent member, so a client that does not see the token sends `2` explicitly. Priority *int `json:"priority,omitempty"` Title string `json:"title"` } @@ -1050,7 +1050,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -1220,7 +1220,7 @@ type CreateIssueRequest struct { // ParentId Creates a typed `parent-child` edge from the new issue to this target. It must not duplicate an edge `dependencies` already spells; naming the same pair twice with two types is a `400`. ParentId *string `json:"parent_id,omitempty"` - // Priority 0 is P0/critical. Absent means the workspace default. + // Priority 0 is P0/critical. Absent means the create default, P2 (`2`), on a server advertising `issues.create.defaultPriority`; such a server never reads an absent member as `0`. An older server stores P0 for an absent member, so a client that does not see the token sends `2` explicitly. Priority *int `json:"priority,omitempty"` // Sender Who sent this, for the message-shaped rows an orchestrator creates. Stored verbatim and interpreted by nothing on this surface. diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index a1cfa73f0..6eb516386 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -434,7 +434,9 @@ func applyCreateItem(prefix string, encoded json.RawMessage, raw map[string]json issue.NoHistory = *wire.NoHistory } - item := &issueops.CreateItem{Key: derefString(wire.Key), Issue: issue} + // An absent `priority` is the role's default (CreateItem.DefaultPriority), + // never a 0 — which is P0 — made up here. + item := &issueops.CreateItem{Key: derefString(wire.Key), Issue: issue, DefaultPriority: wire.Priority == nil} refs, res := applyMetadataRefs(prefix, raw) if res != nil { return nil, res diff --git a/internal/httpapi/batch_create.go b/internal/httpapi/batch_create.go index 063816260..78ffe68e0 100644 --- a/internal/httpapi/batch_create.go +++ b/internal/httpapi/batch_create.go @@ -217,7 +217,9 @@ func batchCreateItem(index int, raw map[string]json.RawMessage) (issueops.BatchC } issue.Labels = *wire.Labels } - item := issueops.BatchCreateItem{Issue: issue} + // An absent `priority` is the role's default (BatchCreateItem.DefaultPriority), + // never a 0 — which is P0 — made up here. + item := issueops.BatchCreateItem{Issue: issue, DefaultPriority: wire.Priority == nil} if wire.Dependencies == nil { return item, nil } diff --git a/internal/httpapi/create.go b/internal/httpapi/create.go index 89901c5c5..ac64727ad 100644 --- a/internal/httpapi/create.go +++ b/internal/httpapi/create.go @@ -233,6 +233,10 @@ func (s *Server) createIssueRequest(w http.ResponseWriter, r *http.Request) (iss ParentID: derefString(wire.ParentId), InheritLabelsFromParent: derefBool(wire.InheritLabelsFromParent), ForceIDPrefix: derefBool(wire.ForceIdPrefix), + // An absent `priority` is the role's default, never a 0 this handler + // would have to make up: 0 is P0, and the document says absent means + // the default (CreateRequest.DefaultPriority). + DefaultPriority: wire.Priority == nil, } // IDPrefix stays ZERO, and its absence is a decision rather than an // omission. It exists because a workspace's own config.yaml prefix wins over diff --git a/internal/httpapi/create_priority_test.go b/internal/httpapi/create_priority_test.go new file mode 100644 index 000000000..211c2ee46 --- /dev/null +++ b/internal/httpapi/create_priority_test.go @@ -0,0 +1,99 @@ +package httpapi + +import ( + "net/http" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// The three create shapes leave the default priority to the ROLE: an absent +// `priority` reaches it as DefaultPriority with a zero Issue.Priority, and an +// explicit `0` reaches it as P0 with the flag off. The handler never makes up +// the number — before, it left Issue.Priority at 0 and an absent member stored +// P0 (critical), against the document's "absent means the default". + +func TestCreateLeavesAnAbsentPriorityToTheRole(t *testing.T) { + for _, tc := range []struct { + name string + body string + wantDefault bool + wantValue int + }{ + {"absent", `{"actor":"alice","title":"t"}`, true, 0}, + {"explicit zero", `{"actor":"alice","title":"t","priority":0}`, false, 0}, + {"explicit three", `{"actor":"alice","title":"t","priority":3}`, false, 3}, + } { + t.Run(tc.name, func(t *testing.T) { + lifecycle := &roleLifecycle{createResult: issueops.CreateResult{Issue: createdIssue("bd-7")}} + ts := newCreateServer(t, lifecycle) + resp := ts.createIssue(t, tc.body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + got := lifecycle.createRequests() + if len(got) != 1 { + t.Fatalf("the role was called %d times, want 1", len(got)) + } + if got[0].DefaultPriority != tc.wantDefault || got[0].Issue.Priority != tc.wantValue { + t.Errorf("role got DefaultPriority=%v Priority=%d, want %v and %d", + got[0].DefaultPriority, got[0].Issue.Priority, tc.wantDefault, tc.wantValue) + } + }) + } +} + +func TestBatchCreateLeavesAnAbsentPriorityToTheRole(t *testing.T) { + creator := &roleBatchCreator{} + ts := newBatchCreateServer(t, creator) + resp := ts.claim(t, batchCreatePath, `{"actor":"alice","items":[ + {"title":"absent"}, + {"title":"zero","priority":0}, + {"title":"three","priority":3} + ]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + reqs := creator.createRequests() + if len(reqs) != 1 || len(reqs[0].Items) != 3 { + t.Fatalf("role calls = %v, want one request of three items", reqs) + } + for i, want := range []struct { + useDefault bool + priority int + }{{true, 0}, {false, 0}, {false, 3}} { + item := reqs[0].Items[i] + if item.DefaultPriority != want.useDefault || item.Issue.Priority != want.priority { + t.Errorf("item %d: DefaultPriority=%v Priority=%d, want %v and %d", + i, item.DefaultPriority, item.Issue.Priority, want.useDefault, want.priority) + } + } +} + +func TestApplyBatchLeavesAnAbsentPriorityToTheRole(t *testing.T) { + applier := &roleBatchApplier{result: issueops.ApplyBatchResult{ + Items: []issueops.ItemResult{ + {Kind: issueops.ItemCreate, IssueID: "bd-1", Changed: true}, + {Kind: issueops.ItemCreate, IssueID: "bd-2", Changed: true}, + }, + }} + ts := newApplyBatchServer(t, applier) + resp := ts.claim(t, batchApplyPath, `{"actor":"alice","items":[ + {"kind":"create","create":{"title":"absent"}}, + {"kind":"create","create":{"title":"zero","priority":0}} + ]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + reqs := applier.requests() + if len(reqs) != 1 || len(reqs[0].Items) != 2 { + t.Fatalf("role calls = %v, want one request of two items", reqs) + } + absent, zero := reqs[0].Items[0].Create, reqs[0].Items[1].Create + if !absent.DefaultPriority || absent.Issue.Priority != 0 { + t.Errorf("absent priority: DefaultPriority=%v Priority=%d, want true and 0", absent.DefaultPriority, absent.Issue.Priority) + } + if zero.DefaultPriority || zero.Issue.Priority != 0 { + t.Errorf("explicit 0: DefaultPriority=%v Priority=%d, want false and 0", zero.DefaultPriority, zero.Issue.Priority) + } +} diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index f0c7b69d9..33af3709b 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -174,6 +174,14 @@ const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" // still edits a template (bd-jkp9v3). const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" +// CapIssuesCreateDefaultPriority is the behavior capability that advertises +// that issues.create, issues.batchCreate and issues.batchApply's create items +// store the create default priority (P2) for a request whose `priority` member +// is ABSENT. An older server predating this token reads an absent priority as +// 0 and stores P0 (critical), so a client that does not see the token MUST +// send the default explicitly rather than omit the member. +const CapIssuesCreateDefaultPriority = "issues.create.defaultPriority" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -935,7 +943,7 @@ func (r route) specPathOf() string { var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, - CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, + CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 0460d6231..99c46b3d9 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -637,7 +637,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // oracle rather than a second call to the code under test. want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", - "issues.batchApply.depAddLineage", "issues.update.allowTemplate") + "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 7465f20b6..a6b5b946b 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -8142,7 +8142,11 @@ components: `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member - unknown. The + unknown, and + `issues.create.defaultPriority`, which announces that a create + (`POST /v0/beads/issues`, `issues:batchCreate` and + `issues:batchApply` create items) with no `priority` member stores + the default P2 rather than P0. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. @@ -9790,7 +9794,12 @@ components: type: integer minimum: 0 maximum: 4 - description: 0 is P0/critical. Absent means the workspace default. + description: >- + 0 is P0/critical. Absent means the create default, P2 (`2`), on a + server advertising `issues.create.defaultPriority`; such a server + never reads an absent member as `0`. An older server stores P0 for + an absent member, so a client that does not see the token sends + `2` explicitly. issue_type: type: string description: >- @@ -10110,7 +10119,12 @@ components: type: integer minimum: 0 maximum: 4 - description: 0 is P0/critical. Absent means the workspace default. + description: >- + 0 is P0/critical. Absent means the create default, P2 (`2`), on a + server advertising `issues.create.defaultPriority`; such a server + never reads an absent member as `0`. An older server stores P0 for + an absent member, so a client that does not see the token sends + `2` explicitly. assignee: type: string maxLength: 255 @@ -12047,7 +12061,12 @@ components: type: integer minimum: 0 maximum: 4 - description: 0 is P0/critical. Absent means the workspace default. + description: >- + 0 is P0/critical. Absent means the create default, P2 (`2`), on a + server advertising `issues.create.defaultPriority`; such a server + never reads an absent member as `0`. An older server stores P0 for + an absent member, so a client that does not see the token sends + `2` explicitly. assignee: type: string maxLength: 255 diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index a2b9d9346..0a733fb39 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -99,6 +99,7 @@ go_test( "completeness_test.go", "composition_test.go", "counts_test.go", + "create_priority_test.go", "credential_source_test.go", "credential_test.go", "dial_test.go", diff --git a/internal/httpclient/batchapplier.go b/internal/httpclient/batchapplier.go index b39c12986..1fd04b1be 100644 --- a/internal/httpclient/batchapplier.go +++ b/internal/httpclient/batchapplier.go @@ -93,6 +93,15 @@ func (b *httpBatchApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBat if err != nil { return issueops.ApplyBatchResult{}, err } + var priorities []**int + for i := range body.Items { + if create := body.Items[i].Create; create != nil { + priorities = append(priorities, &create.Priority) + } + } + if err := b.store.pinCreateDefaultPriority(ctx, priorities...); err != nil { + return issueops.ApplyBatchResult{}, err + } resp, err := b.wire.ApplyBatch(ctx, body) if err != nil { return issueops.ApplyBatchResult{}, applyBatchRefusal(err) @@ -264,11 +273,12 @@ func applyCreateItemBody(item *issueops.CreateItem, actor string) (*apigen.Apply return nil, err } - // Priority is sent ALWAYS, createBody's reason: 0 is P0 and a real request, - // so an absent member — which the server reads as the workspace default — - // would silently reprioritize every critical issue a plan creates. - priority := issue.Priority - out := &apigen.ApplyCreateItem{Title: issue.Title, Priority: &priority} + // Priority is sent whenever the item names one (see wirePriority). + priority, err := wirePriority(issue.Priority, item.DefaultPriority) + if err != nil { + return nil, err + } + out := &apigen.ApplyCreateItem{Title: issue.Title, Priority: priority} setItemString(&out.Key, item.Key) setItemString(&out.Id, issue.ID) diff --git a/internal/httpclient/batchcreator.go b/internal/httpclient/batchcreator.go index ea9b1f7be..7420aaac3 100644 --- a/internal/httpclient/batchcreator.go +++ b/internal/httpclient/batchcreator.go @@ -163,6 +163,13 @@ func (b *httpBatchCreator) CreateBatch(ctx context.Context, req issueops.CreateB } items = append(items, wireItem) } + priorities := make([]**int, len(items)) + for i := range items { + priorities[i] = &items[i].Priority + } + if err := b.store.pinCreateDefaultPriority(ctx, priorities...); err != nil { + return issueops.CreateBatchResult{}, err + } res, err := b.wire.BatchCreateIssues(ctx, apigen.BatchCreateRequest{Actor: req.Actor, Items: items}) if err != nil { @@ -202,12 +209,13 @@ func batchCreateItem(index int, item issueops.BatchCreateItem, actor string) (ap return apigen.BatchCreateItem{}, err } - // Priority is sent ALWAYS, unlike the five optional strings: 0 is P0 and a - // real request, so an absent member — which the server reads as "the - // workspace default" — would silently reprioritize every critical issue in a - // plan. The pointer addresses a local copy, never the caller's field. - priority := item.Issue.Priority - wireItem := apigen.BatchCreateItem{Title: item.Issue.Title, Priority: &priority} + // Priority is sent whenever the item names one, unlike the five optional + // strings (see wirePriority). + priority, err := wirePriority(item.Issue.Priority, item.DefaultPriority) + if err != nil { + return apigen.BatchCreateItem{}, err + } + wireItem := apigen.BatchCreateItem{Title: item.Issue.Title, Priority: priority} setItemString(&wireItem.Description, item.Issue.Description) setItemString(&wireItem.Design, item.Issue.Design) setItemString(&wireItem.AcceptanceCriteria, item.Issue.AcceptanceCriteria) diff --git a/internal/httpclient/bulk_roles_test.go b/internal/httpclient/bulk_roles_test.go index 5bf7a6022..50e48381d 100644 --- a/internal/httpclient/bulk_roles_test.go +++ b/internal/httpclient/bulk_roles_test.go @@ -736,7 +736,9 @@ func TestBatchCreateRefusesEveryMemberTheWireExcludes(t *testing.T) { // dropped it. got := populatableFields(reflect.TypeOf(issueops.BatchCreateItem{})) sort.Strings(got) - want := []string{"Dependencies", "Issue"} + // DefaultPriority is carried as the ABSENCE of the item's `priority` + // member (wirePriority), pinned by TestBatchCreateLeavesPriorityAbsentForTheDefault. + want := []string{"DefaultPriority", "Dependencies", "Issue"} if !reflect.DeepEqual(got, want) { t.Errorf("BatchCreateItem carries %v, want %v.\n"+ "A new member reaches no wire member and no refusal: carry it in batchCreateItem, or refuse it with a ledger row.", got, want) diff --git a/internal/httpclient/create_priority_test.go b/internal/httpclient/create_priority_test.go new file mode 100644 index 000000000..796385f7b --- /dev/null +++ b/internal/httpclient/create_priority_test.go @@ -0,0 +1,215 @@ +package httpclient + +import ( + "context" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// TestBatchCreateLeavesPriorityAbsentForTheDefault pins wirePriority on all +// three create bodies: a request that asks for the default priority sends NO +// `priority` member, so the server's role applies the default — the one place +// it lives — and a request naming 0 sends `0`, because P0 is a real request. +// Asking for the default while naming a priority is refused before the dial, +// the role's own ErrValidation, rather than silently dropping one of the two. +func TestBatchCreateLeavesPriorityAbsentForTheDefault(t *testing.T) { + issue := func(priority int) *types.Issue { + return &types.Issue{Title: "t", IssueType: types.TypeTask, Priority: priority} + } + type encoded struct { + priority *int + err error + } + shapes := map[string]func(priority int, useDefault bool) encoded{ + "createIssue": func(priority int, useDefault bool) encoded { + body, err := createBody(issueops.CreateRequest{Actor: "a", Issue: issue(priority), DefaultPriority: useDefault}) + return encoded{body.Priority, err} + }, + "batchCreateIssues": func(priority int, useDefault bool) encoded { + item, err := batchCreateItem(0, issueops.BatchCreateItem{Issue: issue(priority), DefaultPriority: useDefault}, "a") + return encoded{item.Priority, err} + }, + "applyBatch create item": func(priority int, useDefault bool) encoded { + item, err := applyCreateItemBody(&issueops.CreateItem{Issue: issue(priority), DefaultPriority: useDefault}, "a") + if err != nil { + return encoded{nil, err} + } + return encoded{item.Priority, nil} + }, + } + for name, encode := range shapes { + t.Run(name, func(t *testing.T) { + if got := encode(0, true); got.err != nil || got.priority != nil { + t.Errorf("DefaultPriority: priority member = %v, err = %v; want the member absent", got.priority, got.err) + } + if got := encode(0, false); got.err != nil || got.priority == nil || *got.priority != 0 { + t.Errorf("priority 0: priority member = %v, err = %v; want an explicit 0", got.priority, got.err) + } + if got := encode(3, false); got.err != nil || got.priority == nil || *got.priority != 3 { + t.Errorf("priority 3: priority member = %v, err = %v; want 3", got.priority, got.err) + } + if got := encode(3, true); !errors.Is(got.err, issueops.ErrValidation) { + t.Errorf("DefaultPriority with priority 3: err = %v, want ErrValidation", got.err) + } + }) + } +} + +// createPriorityRoles drives one create through each of the three create roles +// and reports the `priority` member that role put on the wire (nil when it is +// absent), or the role's own error. +func createPriorityRoles() map[string]func(t *testing.T, s *Store, w *stubWire, priority int, useDefault bool) (*int, error) { + issue := func(priority int) *types.Issue { + return &types.Issue{Title: "t", IssueType: types.TypeTask, Priority: priority} + } + return map[string]func(t *testing.T, s *Store, w *stubWire, priority int, useDefault bool) (*int, error){ + "Lifecycle.Create": func(t *testing.T, s *Store, w *stubWire, priority int, useDefault bool) (*int, error) { + lifecycle, err := s.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + if _, err := lifecycle.Create(t.Context(), issueops.CreateRequest{ + Actor: "a", Issue: issue(priority), DefaultPriority: useDefault, + }); err != nil { + return nil, err + } + return w.lastCreate.Priority, nil + }, + "BatchCreator.CreateBatch": func(t *testing.T, s *Store, w *stubWire, priority int, useDefault bool) (*int, error) { + creator, err := s.BatchCreator() + if err != nil { + t.Fatalf("BatchCreator(): %v", err) + } + if _, err := creator.CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: "a", Items: []issueops.BatchCreateItem{{Issue: issue(priority), DefaultPriority: useDefault}}, + }); err != nil { + return nil, err + } + return w.lastBatchCreate.Items[0].Priority, nil + }, + "BatchApplier.ApplyBatch": func(t *testing.T, s *Store, w *stubWire, priority int, useDefault bool) (*int, error) { + applier, err := s.BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + if _, err := applier.ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "a", Items: []issueops.ApplyItem{{ + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{Issue: issue(priority), DefaultPriority: useDefault}, + }}, + }); err != nil { + return nil, err + } + return w.lastApply.Items[0].Create.Priority, nil + }, + } +} + +// showPriority renders a wire `priority` member for a comparison: its value, +// or "absent". +func showPriority(p *int) any { + if p == nil { + return "absent" + } + return *p +} + +// TestCreateSendsTheDefaultPriorityToAServerThatPredatesIt is a NEW client +// against an OLD server, on all three create roles. A handshake without +// issues.create.defaultPriority is a server that reads an absent `priority` as +// 0 and would store P0 (critical) for every create that asked for the +// default, so against it the role sends issueops.DefaultCreatePriority (2) +// explicitly. Against a server that advertises the token the member stays +// absent and the server's role applies the default. An explicit 0 is sent as 0 +// to both. +func TestCreateSendsTheDefaultPriorityToAServerThatPredatesIt(t *testing.T) { + for name, run := range createPriorityRoles() { + for _, server := range []struct { + name string + caps []string + wantDefault any + }{ + {name: "old server", caps: nil, wantDefault: issueops.DefaultCreatePriority}, + {name: "current server", caps: []string{wire.CapIssuesCreateDefaultPriority}, wantDefault: "absent"}, + } { + t.Run(name+"/"+server.name, func(t *testing.T) { + store := func(w *stubWire) *Store { + return New(testTarget(t), w, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: server.caps}) + } + w := &stubWire{} + sent, err := run(t, store(w), w, 0, true) + if err != nil { + t.Fatalf("DefaultPriority: %v", err) + } + if got := showPriority(sent); got != server.wantDefault { + t.Errorf("DefaultPriority: sent priority %v, want %v", got, server.wantDefault) + } + w = &stubWire{} + sent, err = run(t, store(w), w, 0, false) + if err != nil { + t.Fatalf("explicit 0: %v", err) + } + if got := showPriority(sent); got != 0 { + t.Errorf("explicit 0: sent priority %v, want 0", got) + } + }) + } + } +} + +// handshakeWire is a stubWire whose handshake answers with res and err, so a +// test can hand the store a handshake that says nothing, or one that fails. +type handshakeWire struct { + *stubWire + res *apigen.ContextResponse + err error +} + +func (w handshakeWire) ServerContext(context.Context) (*apigen.ContextResponse, error) { + return w.res, w.err +} + +// TestCreateDefaultPriorityWithoutAUsableHandshake covers the handshakes that +// cannot say whether the server applies the default, on all three create +// roles. One that answers no context has no token to read, so the role sends +// issueops.DefaultCreatePriority explicitly, as it does to an older server. One +// that fails fails the create before it is dialed, rather than guessing. A +// request that names its priority never asks, so a failing handshake does not +// stop it. +func TestCreateDefaultPriorityWithoutAUsableHandshake(t *testing.T) { + boom := errors.New("handshake failed") + for name, run := range createPriorityRoles() { + t.Run(name, func(t *testing.T) { + w := &stubWire{} + sent, err := run(t, New(testTarget(t), handshakeWire{stubWire: w}, nil), w, 0, true) + if err != nil { + t.Fatalf("no context, DefaultPriority: %v", err) + } + if got := showPriority(sent); got != issueops.DefaultCreatePriority { + t.Errorf("no context, DefaultPriority: sent priority %v, want %v", got, issueops.DefaultCreatePriority) + } + + w = &stubWire{} + if _, err := run(t, New(testTarget(t), handshakeWire{stubWire: w, err: boom}, nil), w, 0, true); !errors.Is(err, boom) { + t.Errorf("failed handshake, DefaultPriority: err = %v, want the handshake's error", err) + } + if len(w.calls) != 0 { + t.Errorf("failed handshake, DefaultPriority: dialed %v, want nothing", w.calls) + } + + w = &stubWire{} + sent, err = run(t, New(testTarget(t), handshakeWire{stubWire: w, err: boom}, nil), w, 0, false) + if err != nil { + t.Fatalf("failed handshake, explicit 0: %v", err) + } + if got := showPriority(sent); got != 0 { + t.Errorf("failed handshake, explicit 0: sent priority %v, want 0", got) + } + }) + } +} diff --git a/internal/httpclient/lifecycle.go b/internal/httpclient/lifecycle.go index 409be20d3..4475293a2 100644 --- a/internal/httpclient/lifecycle.go +++ b/internal/httpclient/lifecycle.go @@ -63,6 +63,9 @@ func (l *httpLifecycle) Create(ctx context.Context, req issueops.CreateRequest) if err != nil { return issueops.CreateResult{}, err } + if err := l.store.pinCreateDefaultPriority(ctx, &body.Priority); err != nil { + return issueops.CreateResult{}, err + } issue, err := l.wire.CreateIssue(ctx, body) if err != nil { @@ -112,11 +115,11 @@ func createBody(req issueops.CreateRequest) (apigen.CreateIssueRequest, error) { } issue := req.Issue - // Priority is sent ALWAYS, for batchCreateIssues' reason: 0 is P0 and a real - // request, so an absent member — which the server reads as the workspace - // default — would silently reprioritize every critical issue a plan creates. - priority := issue.Priority - body := apigen.CreateIssueRequest{Actor: req.Actor, Title: issue.Title, Priority: &priority} + priority, err := wirePriority(issue.Priority, req.DefaultPriority) + if err != nil { + return apigen.CreateIssueRequest{}, err + } + body := apigen.CreateIssueRequest{Actor: req.Actor, Title: issue.Title, Priority: priority} setItemString(&body.Id, issue.ID) setItemString(&body.Description, issue.Description) @@ -961,3 +964,64 @@ func (s *Store) CloseIssue(ctx context.Context, id, reason, actor, session strin _, err = w.CloseIssue(ctx, id, closeBody(actor, reason, session, true)) return err } + +// wirePriority projects a create's priority onto the wire's optional member. +// +// It is sent WHENEVER THE CALLER NAMED ONE, zero included: 0 is P0 and a real +// request, so an absent member would silently reprioritize critical work. It +// is absent only when the request asks for the default (DefaultPriority), and +// then the server's role applies the default — the one place it lives — rather +// than this client spelling the number, PROVIDED the server says it does: +// pinCreateDefaultPriority fills the member in for one that does not. The +// pointer addresses a local copy, never the caller's field. +func wirePriority(priority int, useDefault bool) (*int, error) { + if err := storageops.ValidateCreatePriority(priority, useDefault); err != nil { + return nil, err + } + if useDefault { + return nil, nil + } + return &priority, nil +} + +// pinCreateDefaultPriority decides, before the dial and from the cached +// handshake, whether the absent priorities among the given wire members may +// stay absent. A server advertising wire.CapIssuesCreateDefaultPriority stores +// the create default for an absent member; an older one reads absent as 0 and +// would store P0 (critical) for every create that asked for the default. So +// against an older server each absent member is set to +// issueops.DefaultCreatePriority explicitly. A request naming every priority +// costs no handshake. +// +// The decision assumes the create reaches the build that answered the +// handshake. The handshake is fetched once per Store and cached for its life +// (one command, in cmd/bd), so a create that lands on an older build — a +// server rolled back under a live Store, or mixed builds behind one URL — +// still goes out with the member absent, and that server stores P0 with no +// error on either side. A capability that gates a member an older server +// refuses is loud on a stale handshake (the `has_spawner` that +// refuseUnservedDepAddLineage gates draws a 400); this one gates the ABSENCE +// of a member every server already reads, so nothing refuses it. +func (s *Store) pinCreateDefaultPriority(ctx context.Context, priorities ...**int) error { + var absent []**int + for _, p := range priorities { + if *p == nil { + absent = append(absent, p) + } + } + if len(absent) == 0 { + return nil + } + snap, err := s.snapshot(ctx) + if err != nil { + return err + } + if snap != nil && slices.Contains(snap.Capabilities, wire.CapIssuesCreateDefaultPriority) { + return nil + } + for _, p := range absent { + priority := issueops.DefaultCreatePriority + *p = &priority + } + return nil +} diff --git a/internal/httpclient/lifecycle_create_test.go b/internal/httpclient/lifecycle_create_test.go index a67b50693..462d291ee 100644 --- a/internal/httpclient/lifecycle_create_test.go +++ b/internal/httpclient/lifecycle_create_test.go @@ -239,7 +239,7 @@ func TestCreateRefusesEveryMemberTheWireExcludes(t *testing.T) { got := populatableFields(reflect.TypeOf(issueops.CreateRequest{})) sort.Strings(got) want := []string{ - "Actor", "Dependencies", "ForceIDPrefix", "IDPrefix", + "Actor", "DefaultPriority", "Dependencies", "ForceIDPrefix", "IDPrefix", "InheritLabelsFromParent", "Issue", "ParentID", "WaitsFor", } if !reflect.DeepEqual(got, want) { diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index e7aa98872..a168048a7 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -269,6 +269,10 @@ func TestServedBatchApplyRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, t.Context(), newServedBatchApplyFixture(t, "hbadot")) } +func TestServedBatchApplyAppliesTheDefaultPriority(t *testing.T) { + conformance.RunBatchApplyAppliesTheDefaultPriority(t, t.Context(), newServedBatchApplyFixture(t, "hbaprio")) +} + // TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed wires the dep_add // HasSpawner/ThreadID lineage conformance case onto the http leg (S5): a // served default Config always advertises CapBatchApplyDepAddLineage (it is diff --git a/internal/httpclient/served_bulk_test.go b/internal/httpclient/served_bulk_test.go index 03bfb4870..f92eb6454 100644 --- a/internal/httpclient/served_bulk_test.go +++ b/internal/httpclient/served_bulk_test.go @@ -628,6 +628,10 @@ func TestServedBatchCreatorEchoesSubSecondTimestamps(t *testing.T) { conformance.RunBatchCreatorEchoesSubSecondTimestamps(t, t.Context(), newServedBatchCreatorFixture(t, "hb12")) } +func TestServedBatchCreatorAppliesTheDefaultPriority(t *testing.T) { + conformance.RunBatchCreatorAppliesTheDefaultPriority(t, t.Context(), newServedBatchCreatorFixture(t, "hbprio")) +} + // The last TWO items of the wire's item vocabulary, both on the EPHEMERAL half // of it and both parked for the reason the eight above are: apigen.BatchCreateItem // carries eight content members and no id and no wisp flag, so an item cannot diff --git a/internal/httpclient/served_lifecycle_create_test.go b/internal/httpclient/served_lifecycle_create_test.go index 6f152753f..5e954503d 100644 --- a/internal/httpclient/served_lifecycle_create_test.go +++ b/internal/httpclient/served_lifecycle_create_test.go @@ -120,6 +120,10 @@ func TestServedLifecycleCreateEchoesSubSecondTimestamps(t *testing.T) { conformance.RunLifecycleCreateEchoesSubSecondTimestamps(t, t.Context(), newServedCreateFixture(t, "hlc6")) } +func TestServedLifecycleCreateAppliesTheDefaultPriority(t *testing.T) { + conformance.RunLifecycleCreateAppliesTheDefaultPriority(t, t.Context(), newServedCreateFixture(t, "hlcprio")) +} + // createParkBead is the bead the create-side parks cite. It is separate from // parkBead — the write-side parks of the close/update wave — because these // retire on different events: two of them wait on the wire growing a diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index a52f81cb1..b0ed49785 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -221,6 +221,15 @@ const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" // (bd-jkp9v3). const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" +// CapIssuesCreateDefaultPriority is the behavior capability announcing that +// the server stores the create default priority for an ABSENT `priority` +// member on issues.create, issues.batchCreate and issues.batchApply create +// items, spelled exactly as httpapi's constant of the same name. The create +// roles read it from the cached handshake before the dial: a server without it +// reads an absent priority as P0, so against it the client sends +// issueops.DefaultCreatePriority explicitly (pinCreateDefaultPriority). +const CapIssuesCreateDefaultPriority = "issues.create.defaultPriority" + // CapExternalDependencies is the CONDITIONAL behavior capability announcing // that the ready, claim and close operations of this server apply bd's // external-dependency policy themselves, spelled exactly as httpapi's constant @@ -361,7 +370,7 @@ func (e *WireRevisionSkewError) Unwrap() error { return ErrWireRevisionSkew } var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, - CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, + CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 01a0ea9cf..8e5abb1dd 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -196,11 +196,11 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if !slices.Equal(behaviorCapabilities, []string{ httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, - httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, + httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, httpapi.CapIssuesCreateDefaultPriority, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit and "+ - "issues.batchApply.depAddLineage and issues.update.allowTemplate tokens", behaviorCapabilities) + "issues.batchApply.depAddLineage and issues.update.allowTemplate and issues.create.defaultPriority tokens", behaviorCapabilities) } } diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go index 69f3514cd..33c2fb62c 100644 --- a/internal/httpclient/write_ledger_test.go +++ b/internal/httpclient/write_ledger_test.go @@ -78,10 +78,19 @@ type carriage struct { // incremental add or remove has no expression. CreateRequest.Issue is the // other, and its excluded shape is a set of MEMBERS rather than a mode. partial string + // absenceOf names the member a field is carried as the ABSENCE of, when + // another field drives that member's presence. CreateRequest.DefaultPriority + // is the population: "use the default" is `priority` left out, the member + // the Issue spread otherwise drives (wirePriority). It drives no member of + // its own, so the wire -> source direction does not count it, but the + // member must still be published. + absenceOf string } func member(name string) carriage { return carriage{member: name} } +func absenceOf(name string) carriage { return carriage{absenceOf: name} } + func spread(names ...string) carriage { return carriage{spread: names} } // members lists every wire member a carriage drives, whichever way it drives @@ -164,6 +173,9 @@ func writeShapes() []writeShape { "Dependencies": member("dependencies"), "WaitsFor": member("waits_for"), "ForceIDPrefix": member("force_id_prefix"), + // DefaultPriority is the ABSENCE of `priority`: the member the + // Issue spread drives is left out (wirePriority). + "DefaultPriority": absenceOf("priority"), }, }, { @@ -512,6 +524,8 @@ func writeShapes() []writeShape { partial: "W-CreateItem.Issue", }, "MetadataRefs": member("metadata_refs"), + // The absence of `priority` (wirePriority). + "DefaultPriority": absenceOf("priority"), }, }, { @@ -652,7 +666,7 @@ func TestEveryWireExcludedWriteMemberCarriesALedgerRow(t *testing.T) { continue } ways := 0 - for _, set := range []bool{how.member != "", len(how.spread) > 0, how.routed != ""} { + for _, set := range []bool{how.member != "", len(how.spread) > 0, how.routed != "", how.absenceOf != ""} { if set { ways++ } @@ -662,6 +676,11 @@ func TestEveryWireExcludedWriteMemberCarriesALedgerRow(t *testing.T) { t.Errorf("%s.%s claims more than one of a body member, a spread and a route; it has exactly one", shape.source.Name(), name) case ways == 0: t.Errorf("%s.%s is in the carried table with neither a member nor a route", shape.source.Name(), name) + case how.absenceOf != "": + if !members[how.absenceOf] { + t.Errorf("%s.%s is carried as the absence of %q, which %s does not publish: %v", + shape.source.Name(), name, how.absenceOf, shape.body.Name(), sortedKeys(members)) + } default: for _, wireMember := range how.members() { if !members[wireMember] { diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index 5d009ca38..e12080dd0 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -275,6 +275,11 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyAppliesTheDefaultPriority(t, ctx, fixture) + }) } // newDoltBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/dolt/batch_creator_contract_test.go b/internal/storage/dolt/batch_creator_contract_test.go index db4e6878e..bd662481f 100644 --- a/internal/storage/dolt/batch_creator_contract_test.go +++ b/internal/storage/dolt/batch_creator_contract_test.go @@ -60,6 +60,9 @@ func TestBatchCreatorContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunBatchCreatorEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunBatchCreatorAppliesTheDefaultPriority(t, ctx, fixture) + }) } // newDoltBatchCreatorFixture composes the frozen role kit with this backend's diff --git a/internal/storage/dolt/lifecycle_create_contract_test.go b/internal/storage/dolt/lifecycle_create_contract_test.go index b8b40def1..c81919692 100644 --- a/internal/storage/dolt/lifecycle_create_contract_test.go +++ b/internal/storage/dolt/lifecycle_create_contract_test.go @@ -39,6 +39,9 @@ func TestLifecycleCreateContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunLifecycleCreateEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunLifecycleCreateAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newDoltLifecycleCreateFixture(t *testing.T, prefix string) (conformance.LifecycleCreateFixture, context.Context, func()) { diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index 6d330a163..343c16083 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -199,6 +199,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("RefusesADottedChildGatedOnItsOwnParent", func(t *testing.T) { conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunBatchApplyAppliesTheDefaultPriority(t, ctx, fixture) + }) } // newEmbeddedBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/embeddeddolt/batch_creator_contract_test.go b/internal/storage/embeddeddolt/batch_creator_contract_test.go index 8bfb53398..bfc283ab6 100644 --- a/internal/storage/embeddeddolt/batch_creator_contract_test.go +++ b/internal/storage/embeddeddolt/batch_creator_contract_test.go @@ -61,6 +61,9 @@ func TestBatchCreatorContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunBatchCreatorEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunBatchCreatorAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newEmbeddedBatchCreatorFixture(t *testing.T, te *testEnv, prefix string) conformance.BatchCreatorFixture { diff --git a/internal/storage/embeddeddolt/lifecycle_create_contract_test.go b/internal/storage/embeddeddolt/lifecycle_create_contract_test.go index 96d586b0d..d05c0ab6f 100644 --- a/internal/storage/embeddeddolt/lifecycle_create_contract_test.go +++ b/internal/storage/embeddeddolt/lifecycle_create_contract_test.go @@ -40,6 +40,9 @@ func TestLifecycleCreateContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunLifecycleCreateEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunLifecycleCreateAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newEmbeddedLifecycleCreateFixture(t *testing.T, te *testEnv, prefix string) conformance.LifecycleCreateFixture { diff --git a/internal/storage/issueops/batch_apply.go b/internal/storage/issueops/batch_apply.go index 0459110eb..0a069f57b 100644 --- a/internal/storage/issueops/batch_apply.go +++ b/internal/storage/issueops/batch_apply.go @@ -238,9 +238,10 @@ func (r *applyBatchRun) applyItem(ctx context.Context, tx *sql.Tx, index int) er // cap. Revisit it with a measurement, not with a copy of ExecuteCreateBatch. func (r *applyBatchRun) applyCreate(ctx context.Context, tx *sql.Tx, index int, item *publicops.CreateItem) error { created, tables, err := ExecuteCreate(ctx, tx, publicops.CreateRequest{ - Actor: r.plan.Actor, - Issue: item.Issue, - ForceIDPrefix: r.plan.ForceIDPrefix, + Actor: r.plan.Actor, + Issue: item.Issue, + ForceIDPrefix: r.plan.ForceIDPrefix, + DefaultPriority: item.DefaultPriority, }) if err != nil { return &publicops.ItemError{Index: index, Kind: publicops.ItemCreate, Key: item.Key, Err: err} diff --git a/internal/storage/issueops/create_batch.go b/internal/storage/issueops/create_batch.go index 4739f470d..29c660ccc 100644 --- a/internal/storage/issueops/create_batch.go +++ b/internal/storage/issueops/create_batch.go @@ -17,10 +17,11 @@ func CloneCreateBatchRequest(request publicops.CreateBatchRequest) publicops.Cre clone := request clone.Items = make([]publicops.BatchCreateItem, len(request.Items)) for i, item := range request.Items { - clone.Items[i] = publicops.BatchCreateItem{ - Issue: clonePublicIssue(item.Issue), - Dependencies: append([]publicops.CreateDependency(nil), item.Dependencies...), - } + // Start from the item itself so every scalar member (DefaultPriority + // today) survives the clone; only the reference members are deepened. + clone.Items[i] = item + clone.Items[i].Issue = clonePublicIssue(item.Issue) + clone.Items[i].Dependencies = append([]publicops.CreateDependency(nil), item.Dependencies...) } return clone } @@ -53,10 +54,11 @@ func ValidateCreateBatchRequest(request publicops.CreateBatchRequest) error { // an item through it, so no item restates CreateRequest's field rules. func CreateBatchItemRequest(request publicops.CreateBatchRequest, item publicops.BatchCreateItem) publicops.CreateRequest { return publicops.CreateRequest{ - Actor: request.Actor, - Issue: item.Issue, - Dependencies: item.Dependencies, - ForceIDPrefix: request.ForceIDPrefix, + Actor: request.Actor, + Issue: item.Issue, + Dependencies: item.Dependencies, + ForceIDPrefix: request.ForceIDPrefix, + DefaultPriority: item.DefaultPriority, } } diff --git a/internal/storage/issueops/public_create.go b/internal/storage/issueops/public_create.go index 693edac9e..d80952eea 100644 --- a/internal/storage/issueops/public_create.go +++ b/internal/storage/issueops/public_create.go @@ -47,9 +47,24 @@ func ValidatePublicCreateRequest(request publicops.CreateRequest) error { if err := types.CheckFieldLen("parent ID", request.ParentID); err != nil { return publicCreateValidationError(fmt.Errorf("create: %w", err)) } + if err := ValidateCreatePriority(request.Issue.Priority, request.DefaultPriority); err != nil { + return err + } return validatePublicCreateDependencies(request) } +// ValidateCreatePriority refuses a create that asks for the default priority +// (CreateRequest.DefaultPriority, and the batch items' copies of it) while its +// issue also names one: the two would disagree about what was asked for. It is +// exported so a wire client refuses the same request before the dial instead +// of dropping the member it cannot send both of. +func ValidateCreatePriority(issuePriority int, useDefault bool) error { + if useDefault && issuePriority != 0 { + return publicCreateValidationError(fmt.Errorf("create: DefaultPriority is set but the issue names priority %d", issuePriority)) + } + return nil +} + // PreparePublicCreateRequest snapshots, normalizes, and validates a public // create request using the supplied configuration. func PreparePublicCreateRequest(request publicops.CreateRequest, context PublicCreateContext) (publicops.CreateRequest, error) { @@ -61,6 +76,13 @@ func PreparePublicCreateRequest(request publicops.CreateRequest, context PublicC if issue.Status == "" { issue.Status = types.StatusOpen } + // THE ONE PLACE the create priority default is applied: every + // Lifecycle.Create, BatchCreator item and BatchApplier create item on every + // backend is prepared here, so the CLI and the HTTP handlers only say + // "absent" and never spell the number. + if request.DefaultPriority { + issue.Priority = publicops.DefaultCreatePriority + } if issue.ID != "" && !request.ForceIDPrefix { // The caller's prefix wins when it supplied one; see // CreateRequest.IDPrefix for why a front door may know better than the @@ -78,6 +100,9 @@ func PreparePublicCreateRequest(request publicops.CreateRequest, context PublicC } prepared := request prepared.Issue = issue + // The default is spent: the prepared issue now carries the priority, and + // the flag would contradict it on the re-validation below. + prepared.DefaultPriority = false if prepared.WaitsFor != nil && prepared.WaitsFor.Gate == "" { prepared.WaitsFor.Gate = string(types.WaitsForAllChildren) } diff --git a/internal/storage/uow/batch_applier.go b/internal/storage/uow/batch_applier.go index 8562f63b7..c566ef2db 100644 --- a/internal/storage/uow/batch_applier.go +++ b/internal/storage/uow/batch_applier.go @@ -218,9 +218,10 @@ func (r *uowApplyRun) applyCreate(ctx context.Context, index int, item *publicop return err } prepared, err := storageissueops.PreparePublicCreateRequest(publicops.CreateRequest{ - Actor: r.plan.Actor, - Issue: item.Issue, - ForceIDPrefix: r.plan.ForceIDPrefix, + Actor: r.plan.Actor, + Issue: item.Issue, + ForceIDPrefix: r.plan.ForceIDPrefix, + DefaultPriority: item.DefaultPriority, }, storageissueops.PublicCreateContext{ IssuePrefix: createContext.IssuePrefix, AllowedPrefixes: createContext.AllowedPrefixes, diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index edfa1af3d..8d4158ac6 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -169,6 +169,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("RefusesADottedChildGatedOnItsOwnParent", func(t *testing.T) { conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunBatchApplyAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newUOWBatchApplyFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchApplyFixture { diff --git a/internal/storage/uow/batch_creator_contract_test.go b/internal/storage/uow/batch_creator_contract_test.go index 0bc5f74fb..99621f472 100644 --- a/internal/storage/uow/batch_creator_contract_test.go +++ b/internal/storage/uow/batch_creator_contract_test.go @@ -61,6 +61,9 @@ func TestBatchCreatorContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunBatchCreatorEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunBatchCreatorAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newUOWBatchCreatorFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchCreatorFixture { diff --git a/internal/storage/uow/lifecycle_create_contract_test.go b/internal/storage/uow/lifecycle_create_contract_test.go index 19e1d16b5..e32d4a85f 100644 --- a/internal/storage/uow/lifecycle_create_contract_test.go +++ b/internal/storage/uow/lifecycle_create_contract_test.go @@ -40,6 +40,9 @@ func TestLifecycleCreateContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunLifecycleCreateEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunLifecycleCreateAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newUOWLifecycleCreateFixture(t *testing.T, ctx context.Context, prefix string) conformance.LifecycleCreateFixture { diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index 7b3634a08..71116da50 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -100,6 +100,9 @@ type CreateItem struct { // A caller reading the event stream sees a create and then an update, not // one create carrying values nothing could have known yet. MetadataRefs map[string]Ref + // DefaultPriority is CreateRequest.DefaultPriority for this item: the item + // has no priority, and the create stores DefaultCreatePriority. + DefaultPriority bool } // UpdateItem patches one existing issue, under UpdateRequest's rules. diff --git a/issueops/batchcreator.go b/issueops/batchcreator.go index e75e1fbbc..f3fbe8316 100644 --- a/issueops/batchcreator.go +++ b/issueops/batchcreator.go @@ -43,6 +43,8 @@ type BatchCreateItem struct { // referred to at all, because nothing can name an id that does not exist // yet. Dependencies []CreateDependency + // DefaultPriority is CreateRequest.DefaultPriority for this item. + DefaultPriority bool } // CreateBatchRequest describes one creation of many issues. diff --git a/issueops/issueops.go b/issueops/issueops.go index 3d5588e2b..cb55aff60 100644 --- a/issueops/issueops.go +++ b/issueops/issueops.go @@ -224,8 +224,23 @@ type CreateRequest struct { // is handed. Without it the two `bd create` routes disagree about which // ids a workspace may mint. IDPrefix string + // DefaultPriority says the caller has no priority for the issue, and the + // create stores DefaultCreatePriority. It exists because Issue.Priority + // cannot say "absent": its zero value is P0, a real and urgent priority, so + // a caller that leaves it unset asks for critical work. The default is + // applied here, once, for every route — the CLI's `bd create` without + // --priority and an HTTP create without `priority` both set this rather + // than spelling the number themselves. + // + // Issue.Priority must be zero when it is set; a request that sets both is + // ErrValidation, because the two would disagree about what was asked for. + DefaultPriority bool } +// DefaultCreatePriority is the priority a create stores when its request sets +// DefaultPriority (CreateRequest, BatchCreateItem, CreateItem): P2, medium. +const DefaultCreatePriority = 2 + // UpdateRequest describes an issue update. type UpdateRequest struct { Actor, IssueID string From ff4f46e13702d93fd0c07c3f6dcb72c72d08bf68 Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 10 Oct 2026 02:00:02 +0000 Subject: [PATCH 028/151] fix(bazel): go_test_pinned_shard.sh runs under bash 3.2 (macOS Main red since #7413) (#7423) The pinned-shard wrapper kept a `declare -A shard_of` map whose only use was the "pinned twice" check. Apple's /bin/bash is 3.2, which has no associative arrays, so on the main workflow's macOS Test job scripts/pinned_shards_test.go (which runs the wrapper through the host `bash`) failed every sharded case with `declare: -A: invalid option` (exit 2). The job has been red on every main push since #7413 landed (runs 37890739870, 37900573128, 37902968742); the Bazel lanes run the wrapper on Linux bash 5 and never saw it. The check is now a membership test on the "|"-joined `all` string, which the manifest's name pattern (^Test[A-Za-z0-9_]*$) keeps free of "|". Nothing else changes: the pinned-twice refusal still fires (TestPinnedShardWrapperRejectsBadManifests/test_pinned_twice). Verified: go test -run TestPinnedShard ./scripts with PATH limited to /usr/bin:/bin (bash 3.2.57): 5 PASS; the whole ./scripts package under bash 5.3: ok; shellcheck -s bash and `/bin/bash -n` clean. Co-authored-by: Claude Fable 5.1 --- tools/bazel/go_test_pinned_shard.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/bazel/go_test_pinned_shard.sh b/tools/bazel/go_test_pinned_shard.sh index d1c2b5d7d..642526835 100755 --- a/tools/bazel/go_test_pinned_shard.sh +++ b/tools/bazel/go_test_pinned_shard.sh @@ -43,7 +43,10 @@ if [[ "$total" -le 1 ]]; then fi index="${TEST_SHARD_INDEX:?TEST_TOTAL_SHARDS is set but TEST_SHARD_INDEX is not}" -declare -A shard_of=() +# No associative array (bash >= 4): macOS runs this under /bin/bash 3.2 +# (scripts/pinned_shards_test.go on the main workflow's macOS job), so the +# "pinned twice" check is a membership test on the "|"-joined names, which +# the name pattern above keeps free of "|". declare -a pinned_by_shard=() pinned_shards=0 all="" @@ -57,11 +60,10 @@ while IFS= read -r line || [[ -n "$line" ]]; do echo "$manifest:$lineno: want \" \", got: $line" >&2 exit 1 fi - if [[ -n "${shard_of[$name]:-}" ]]; then + if [[ "|$all|" == *"|$name|"* ]]; then echo "$manifest:$lineno: $name is pinned twice" >&2 exit 1 fi - shard_of[$name]="$shard" pinned_by_shard[shard]="${pinned_by_shard[shard]:+${pinned_by_shard[shard]}|}$name" all="${all:+$all|}$name" ((shard > pinned_shards)) && pinned_shards="$shard" From 1cf193bc9ff8f97acb477a3a6a2d67e0ede1c8fe Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 10 Oct 2026 02:39:59 +0000 Subject: [PATCH 029/151] fix(testutil): sweep Dolt test containers whose owner process exited (#7028) * fix(testutil): sweep Dolt test containers whose owner process exited (wy-4hknjt) Every Dolt container the test harness starts now carries two labels naming its owner: the test binary's pid and the host it runs on. Before a process starts its first container, the harness lists the containers labelled for this host and removes the ones whose owner pid no longer exists. In-process cleanup (t.Cleanup, TerminateDoltContainer after m.Run) never runs when a test binary is killed or panics on `go test -timeout`, and a box running with TESTCONTAINERS_RYUK_DISABLED=true has no reaper either. Each such run stranded one idle dolt-sql-server holding 100-470 MiB; 23 were standing on one loaded box after a single evening of killed runs. The sweep bounds the pile at "runs since the last run". Safety under concurrent test processes on one host: a container is removed only when kill(pid, 0) reports ESRCH. A live owner, a pid reused by another live process, or a pid this user may not signal (EPERM) all keep the container. Containers without the owner labels (older harnesses, other tools) are never touched, and the host label keeps a remote DOCKER_HOST's containers, whose pids mean nothing here, out of the sweep. The local (BEADS_TEST_DOLT_SERVER=local) backend is unaffected. Tests: the decision table (dead owners only; missing, non-numeric and non-positive pids skipped without probing), pidAlive on self and on an impossible pid, the label option stamping this process, and a Docker test that starts a dead-owned and a live-owned container, sweeps, and checks only the dead-owned one is gone. Co-Authored-By: Claude Fable 5.1 * build(bazel): sync BUILD files for testutil dolt sweep test wy-4hknjt: apply the gazelle bazel-sync patch from run 36720164405 so internal/testutil/BUILD.bazel carries the testcontainers deps. Co-Authored-By: Claude Fable 5.1 * fix(testutil): key the Dolt container sweep on the owner's PID namespace Maintainer fixup for #7028, review iterations 1 and 2 and pre-approval CI. A pid names a process only inside its PID namespace. The sweep removed a labelled container whenever kill(owner-pid, 0) returned ESRCH, so a sweeper in another namespace that shares the hostname and the Docker daemon reads every live owner as dead. That is the default shape of the maintainers' agent fleet: each session runs under bwrap --unshare-pid without --unshare-uts. Every session therefore reports one hostname, sees only its own handful of pids, and talks to one /var/run/docker.sock, with the Docker backend as the test default. On one such box on 2026-10-06, three live dolt-sql-server test containers from three test runs shared the daemon. With the original sweep merged, the first container start of a test binary in any other session would have removed them mid-test. Reproduced and fixed by running the sweep from a container (its own pid namespace, same hostname, same daemon) against a container labelled with a pid that was alive in the labelling session: - at 5978cedaa2: "removed stale Dolt test container ... (owner process exited)" while the owner was alive; - with this change: the row is listed and kept. Changes: - Stamp a third owner label, owner-pidns: the /proc/self/ns/pid link on Linux and "" elsewhere. staleContainers considers only rows from the sweeper's own namespace. On Linux the sweep is skipped when that link cannot be read. A start-time label alone would not fix this: across namespaces it would turn a pid collision into "dead" as well. An owner whose hostname or namespace cannot be read is stamped "", which no sweep on the host matches, so its container leaks instead. - docker rm -f -v: the image declares VOLUME /var/lib/dolt, and plain rm -f left that anonymous volume behind for every swept container, unlike testcontainers' own RemoveVolumes teardown. - Bound the sweep with a 30s context, plus WaitDelay on each docker command. Without WaitDelay, killing the docker process on timeout does not unblock Output() while a child it started still holds the pipe. With a forking docker shim on PATH, the call returns after 3s with this change and is still blocked after 15s without either part. - Extract parseOwnedRows and table-test it: empty fields, CRLF, trailing newline. - TestSweepStaleDoltContainers: the dead-owned container's owner is a child process that lives until both containers are up and the volumes are read, and is then killed and reaped (a zombie still reads as alive). A sibling test binary's sweep (go test ./... runs them in parallel) removes a container as soon as its owner reads as dead, and a never-live owner pid exposed the container for its whole start. The test checks only the end state for that container and its volume, polled for up to sweepTimeout, because the sibling may get there first or still be removing it when our rm fails. Reproduced with a second test process in the same PID namespace running the real sweep every 100ms alongside TestSweepStaleDoltContainers: - with a never-live owner pid: "starting Dolt container: ... No such container", the container removed inside dolt.Run; - with the child owner but no poll: 1 of 4 runs failed with "dead-owned container ... still exists after the sweep", after the sibling's rm -f -v won and ours exited 1; - with both: 6 of 6 runs pass, 2 of them through that same path. Without the reap, or without -v, the test fails. A container leaked from another namespace is now left to a sweep from that namespace, or to Ryuk. That is the conservative side of the trade. Pre-approval CI: main's //internal/testutil:testutil_dolt_test (#7328) runs this package with BEADS_TEST_DOLT_SERVER=local and BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 in the required dolt-server lane, whose remote workers have no docker. Under the local backend checkDolt reports ready once the pinned dolt CLI resolves, so TestSweepStaleDoltContainers went on to dolt.Run and failed ("failed to create Docker provider"). It now gates like TestDoltServerFingerprint's container subtest: under the local backend it runs only when docker and the image are available, and skips otherwise. On the merge with main b1fe68e63a, with docker unreachable, that package run (race, -parallel 4) failed only this test; with this change it passes with the test skipped, and with docker available the test still runs and passes under either backend. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- internal/testutil/BUILD.bazel | 66 ++++++- internal/testutil/container_provider.go | 2 + internal/testutil/testdoltreap.go | 209 +++++++++++++++++++++ internal/testutil/testdoltreap_test.go | 240 ++++++++++++++++++++++++ internal/testutil/testdoltserver.go | 4 + 5 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 internal/testutil/testdoltreap.go create mode 100644 internal/testutil/testdoltreap_test.go diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 03563cbde..82a658d83 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "suite_temp.go", "testdoltbranch.go", "testdoltcommon.go", + "testdoltreap.go", "testdoltserver.go", "testdoltserver_windows.go", "tmpfs.go", @@ -114,6 +115,7 @@ go_test( "localdoltserver_test.go", "suite_temp_test.go", "testdoltbranch_test.go", + "testdoltreap_test.go", "testdoltserver_env_test.go", "testdoltserver_unreachable_test.go", "tmpfs_test.go", @@ -123,7 +125,69 @@ go_test( deps = [ "//internal/storage/doltutil", "@com_github_go_sql_driver_mysql//:mysql", - ], + ] + select({ + "@rules_go//go/platform:aix": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:android": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:darwin": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:dragonfly": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:freebsd": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:illumos": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:ios": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:js": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:linux": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:netbsd": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:openbsd": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:osx": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:plan9": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:qnx": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:solaris": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "//conditions:default": [], + }), ) # testutil_test on the dolt-server lane: the package's own tests of the diff --git a/internal/testutil/container_provider.go b/internal/testutil/container_provider.go index b6573eb41..6f461a776 100644 --- a/internal/testutil/container_provider.go +++ b/internal/testutil/container_provider.go @@ -39,9 +39,11 @@ func NewContainerProvider() (*ContainerProvider, error) { ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() + reapStaleDoltContainers() ctr, err := dolt.Run(ctx, DoltDockerImage, dolt.WithDatabase("beads_test"), testcontainers.WithEnv(map[string]string{"DOLT_ROOT_HOST": "%"}), + ownerLabels(), ) if err != nil { return nil, fmt.Errorf("starting Dolt container: %w", err) diff --git a/internal/testutil/testdoltreap.go b/internal/testutil/testdoltreap.go new file mode 100644 index 000000000..8cb8cf266 --- /dev/null +++ b/internal/testutil/testdoltreap.go @@ -0,0 +1,209 @@ +//go:build !windows + +package testutil + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/testcontainers/testcontainers-go" +) + +// Stale-container sweep for the Dolt test containers this package starts. +// +// Every container started here carries three labels naming its owner: the +// test binary's pid, the PID namespace that pid belongs to, and the host it +// runs on. Before the first container of a process starts, +// reapStaleDoltContainers lists the containers labeled for this host and +// removes the ones whose owner ran in this process's PID namespace and no +// longer exists. +// +// Why: in-process cleanup (t.Cleanup, TerminateDoltContainer after m.Run) +// never runs when a test binary is killed, or panics on `go test -timeout`, +// and a box running with TESTCONTAINERS_RYUK_DISABLED=true has no reaper +// either. Each such run stranded one idle dolt-sql-server holding 100-470 MiB +// (23 at once on one loaded box, 2026-09-29). The sweep bounds the pile at +// "runs since the last run" instead of "forever". +// +// Safety under concurrent test processes sharing one Docker daemon: a +// container is removed only when its owner ran in this process's PID +// namespace and kill(pid, 0) reports ESRCH. A live owner, a pid reused by +// another live process, or a pid this user may not signal (EPERM) all keep +// the container. A pid means nothing outside its namespace, and sandboxed +// runners (bwrap --unshare-pid, a container without --pid=host) can share the +// hostname and the daemon while seeing none of each other's processes, so a +// container owned from another namespace is never a candidate: if it leaked, +// only a sweep from its own namespace removes it. The namespace label is the +// /proc/self/ns/pid link on Linux, unique among one machine's live +// namespaces, and "" elsewhere; a new namespace that reuses a dead one's +// inode only sees containers whose owners have all exited. Containers without +// the owner labels (older harnesses, other tools) are never touched, and the +// host label keeps a remote DOCKER_HOST's containers, whose pids mean nothing +// here, out of the sweep. + +const ( + ownerPIDLabel = "com.gastownhall.beads.testutil.owner-pid" + ownerPIDNSLabel = "com.gastownhall.beads.testutil.owner-pidns" + ownerHostLabel = "com.gastownhall.beads.testutil.owner-host" +) + +// sweepTimeout bounds the whole sweep, so a wedged daemon or container delays +// the first container start of a process instead of hanging it. +const sweepTimeout = 30 * time.Second + +var reapStaleOnce sync.Once + +// ownerLabels returns the container option that stamps the owner labels. +func ownerLabels() testcontainers.CustomizeRequestOption { + return testcontainers.WithLabels(ownerLabelValues(os.Getpid())) +} + +// ownerLabelValues returns the owner labels for a container owned by pid, a +// process in this process's PID namespace on this host. A hostname, or a Linux +// PID namespace, that cannot be read is stamped as "", which no sweep on this +// host matches: the container leaks rather than risk removal while pid lives. +func ownerLabelValues(pid int) map[string]string { + host, _ := os.Hostname() + ns, _ := pidNamespace() + return map[string]string{ + ownerPIDLabel: strconv.Itoa(pid), + ownerPIDNSLabel: ns, + ownerHostLabel: host, + } +} + +// pidNamespace identifies the PID namespace that os.Getpid and kill(2) +// resolve pids in: the /proc/self/ns/pid link target on Linux, such as +// "pid:[4026531836]", and "" elsewhere. +func pidNamespace() (string, error) { + if runtime.GOOS != "linux" { + return "", nil + } + return os.Readlink("/proc/self/ns/pid") +} + +// ownedContainer is one `docker ps` row: the container id and the raw values +// of its owner-pid and owner-pidns labels. +type ownedContainer struct { + id string + pid string + pidNS string +} + +// staleContainers returns the ids of the rows owned from PID namespace ns +// whose owner pid is dead according to alive. Rows owned from another +// namespace, and rows whose pid label is missing, non-numeric or +// non-positive, are kept: their owner cannot be checked from here, so nothing +// is done. +func staleContainers(rows []ownedContainer, ns string, alive func(pid int) bool) []string { + var stale []string + for _, r := range rows { + if r.pidNS != ns { + continue + } + pid, err := strconv.Atoi(strings.TrimSpace(r.pid)) + if err != nil || pid <= 0 { + continue + } + if alive(pid) { + continue + } + stale = append(stale, r.id) + } + return stale +} + +// pidAlive reports whether a process with the given pid exists. Only ESRCH +// counts as dead; EPERM means the process exists but belongs to someone else. +func pidAlive(pid int) bool { + err := syscall.Kill(pid, 0) + return !errors.Is(err, syscall.ESRCH) +} + +// dockerCommand returns a docker CLI command bound to ctx. WaitDelay lets it +// return once ctx is done even if a process the docker binary started, such +// as a wrapper script's child, still holds its output pipe open. +func dockerCommand(ctx context.Context, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "docker", args...) + cmd.WaitDelay = time.Second + return cmd +} + +// listOwnedContainers returns every container (running or not) that carries +// this package's owner-host label for host. +func listOwnedContainers(ctx context.Context, host string) ([]ownedContainer, error) { + out, err := dockerCommand(ctx, "ps", "-a", "--no-trunc", + "--filter", "label="+ownerHostLabel+"="+host, + "--format", fmt.Sprintf("{{.ID}}\t{{.Label %q}}\t{{.Label %q}}", ownerPIDLabel, ownerPIDNSLabel), + ).Output() + if err != nil { + return nil, fmt.Errorf("docker ps: %w", err) + } + return parseOwnedRows(string(out)), nil +} + +// parseOwnedRows parses listOwnedContainers' `docker ps` output: one +// "idpidpidns" line per container, where a missing label prints as +// an empty field. Lines of any other shape are dropped. +func parseOwnedRows(out string) []ownedContainer { + var rows []ownedContainer + for _, line := range strings.Split(out, "\n") { + f := strings.Split(strings.TrimSuffix(line, "\r"), "\t") + if len(f) != 3 || f[0] == "" { + continue + } + rows = append(rows, ownedContainer{id: f[0], pid: f[1], pidNS: f[2]}) + } + return rows +} + +// sweepStaleDoltContainers removes every container on this host whose owner +// ran in this PID namespace and has exited, and returns the ids it removed. +// It is best effort: a docker failure is reported on stderr and leaves the +// containers alone. +func sweepStaleDoltContainers() []string { + host, err := os.Hostname() + if err != nil || host == "" { + return nil + } + ns, err := pidNamespace() + if err != nil { + fmt.Fprintf(os.Stderr, "testutil: stale Dolt container sweep skipped: %v\n", err) + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), sweepTimeout) + defer cancel() + rows, err := listOwnedContainers(ctx, host) + if err != nil { + fmt.Fprintf(os.Stderr, "testutil: stale Dolt container sweep skipped: %v\n", err) + return nil + } + var removed []string + for _, id := range staleContainers(rows, ns, pidAlive) { + // -v also removes the anonymous volume behind the image's + // VOLUME /var/lib/dolt, as testcontainers' own teardown does. + if err := dockerCommand(ctx, "rm", "-f", "-v", id).Run(); err != nil { + fmt.Fprintf(os.Stderr, "testutil: removing stale Dolt container %s: %v\n", id, err) + continue + } + fmt.Fprintf(os.Stderr, "testutil: removed stale Dolt test container %s (owner process exited)\n", id[:12]) + removed = append(removed, id) + } + return removed +} + +// reapStaleDoltContainers runs sweepStaleDoltContainers once per process. It +// is called before the first Dolt container of a process starts, so the cost +// of one `docker ps` is paid once and only by runs that use Docker at all. +func reapStaleDoltContainers() { + reapStaleOnce.Do(func() { sweepStaleDoltContainers() }) +} diff --git a/internal/testutil/testdoltreap_test.go b/internal/testutil/testdoltreap_test.go new file mode 100644 index 000000000..75a0b1b16 --- /dev/null +++ b/internal/testutil/testdoltreap_test.go @@ -0,0 +1,240 @@ +//go:build !windows + +package testutil + +import ( + "context" + "math" + "os" + "os/exec" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/dolt" +) + +// testPIDNS stands in for this process's PID namespace in the pure tests. +const testPIDNS = "pid:[4026531836]" + +func TestStaleContainers_OnlyDeadOwnersAreStale(t *testing.T) { + alive := func(pid int) bool { return pid == 100 || pid == 300 } + rows := []ownedContainer{ + {id: "live-a", pid: "100", pidNS: testPIDNS}, + {id: "dead-b", pid: "200", pidNS: testPIDNS}, + {id: "live-c", pid: " 300 ", pidNS: testPIDNS}, + {id: "dead-d", pid: "400", pidNS: testPIDNS}, + {id: "no-label", pid: "", pidNS: testPIDNS}, + {id: "garbage", pid: "abc", pidNS: testPIDNS}, + {id: "zero", pid: "0", pidNS: testPIDNS}, + {id: "negative", pid: "-7", pidNS: testPIDNS}, + } + got := staleContainers(rows, testPIDNS, alive) + want := []string{"dead-b", "dead-d"} + if !slices.Equal(got, want) { + t.Fatalf("staleContainers = %v, want %v", got, want) + } +} + +func TestStaleContainers_NeverCallsAliveForUnownedRows(t *testing.T) { + alive := func(pid int) bool { + t.Fatalf("alive called for pid %d; rows without a valid owner pid must be skipped", pid) + return true + } + rows := []ownedContainer{ + {id: "x", pid: "", pidNS: testPIDNS}, + {id: "y", pid: "nope", pidNS: testPIDNS}, + {id: "z", pid: "0", pidNS: testPIDNS}, + } + if got := staleContainers(rows, testPIDNS, alive); got != nil { + t.Fatalf("staleContainers = %v, want nil", got) + } +} + +// A pid names a process only inside its PID namespace: checked from another +// one, a live owner reads as ESRCH. Rows owned from another namespace must +// never reach alive. +func TestStaleContainers_SkipsOtherPIDNamespaces(t *testing.T) { + alive := func(pid int) bool { + t.Fatalf("alive called for pid %d owned from another PID namespace", pid) + return true + } + rows := []ownedContainer{ + {id: "sibling-sandbox", pid: "200", pidNS: "pid:[4026546206]"}, + {id: "no-pidns-label", pid: "200", pidNS: ""}, + } + if got := staleContainers(rows, testPIDNS, alive); got != nil { + t.Fatalf("staleContainers = %v, want nil", got) + } +} + +func TestParseOwnedRows(t *testing.T) { + for _, tc := range []struct { + name string + out string + want []ownedContainer + }{ + {name: "no containers", out: "", want: nil}, + { + name: "trailing newline", + out: "aaa\t12\tpid:[7]\nbbb\t34\tpid:[7]\n", + want: []ownedContainer{{id: "aaa", pid: "12", pidNS: "pid:[7]"}, {id: "bbb", pid: "34", pidNS: "pid:[7]"}}, + }, + {name: "CRLF", out: "aaa\t12\tpid:[7]\r\n", want: []ownedContainer{{id: "aaa", pid: "12", pidNS: "pid:[7]"}}}, + // A missing label prints as an empty field. An empty pidns is what + // every owner outside Linux stamps, so the row must survive parsing. + {name: "empty pidns label", out: "aaa\t12\t\n", want: []ownedContainer{{id: "aaa", pid: "12"}}}, + {name: "empty labels", out: "aaa\t\t\n", want: []ownedContainer{{id: "aaa"}}}, + {name: "malformed lines", out: "aaa\t12\nbbb\n\t12\tpid:[7]\n", want: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := parseOwnedRows(tc.out); !slices.Equal(got, tc.want) { + t.Errorf("parseOwnedRows(%q) = %+v, want %+v", tc.out, got, tc.want) + } + }) + } +} + +func TestOwnerLabels_StampThisProcess(t *testing.T) { + var req testcontainers.GenericContainerRequest + if err := ownerLabels().Customize(&req); err != nil { + t.Fatalf("ownerLabels: %v", err) + } + if got, want := req.Labels[ownerPIDLabel], strconv.Itoa(os.Getpid()); got != want { + t.Errorf("%s = %q, want %q", ownerPIDLabel, got, want) + } + ns, err := pidNamespace() + if err != nil { + t.Fatalf("pidNamespace: %v", err) + } + if got := req.Labels[ownerPIDNSLabel]; got != ns { + t.Errorf("%s = %q, want %q", ownerPIDNSLabel, got, ns) + } + host, _ := os.Hostname() + if got := req.Labels[ownerHostLabel]; got != host { + t.Errorf("%s = %q, want %q", ownerHostLabel, got, host) + } +} + +func TestPidAlive(t *testing.T) { + if !pidAlive(os.Getpid()) { + t.Fatal("pidAlive(self) = false") + } + // No process can have a pid this large on Linux (pid_max <= 2^22) or + // macOS (PID_MAX 99998): kill(2) reports ESRCH. + if pidAlive(math.MaxInt32) { + t.Fatalf("pidAlive(%d) = true, want false", math.MaxInt32) + } +} + +// TestSweepStaleDoltContainers starts one Dolt container owned by a child +// process and one owned by this process, ends the child, sweeps, and checks +// that only the container whose owner exited and its volume are gone. +func TestSweepStaleDoltContainers(t *testing.T) { + if hasTestSkip("dolt") { + t.Skip("skipping: Dolt tests skipped (BEADS_TEST_SKIP=dolt)") + } + // The sweep acts on containers, so it needs docker whichever backend + // BEADS_TEST_DOLT_SERVER selects; under the local one checkDolt only finds + // the dolt CLI (the dolt-server lane's remote workers have no docker). + if !useLocalDoltServer() { + if state := checkDolt(); state != doltReady { + skipOrFailDoltUnavailable(t, state) + } + } else if !isDockerAvailable() || !isDoltImageCached() { + t.Skipf("container backend not available on this host (docker + %s)", DoltDockerImage) + } + if host, err := os.Hostname(); err != nil || host == "" { + t.Skipf("hostname unavailable: %v", err) + } + + start := func(t *testing.T, ownerPID int) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) + defer cancel() + ctr, err := dolt.Run(ctx, DoltDockerImage, + dolt.WithDatabase("beads_test"), + testcontainers.WithEnv(map[string]string{"DOLT_ROOT_HOST": "%"}), + testcontainers.WithLabels(ownerLabelValues(ownerPID)), + ) + if err != nil { + t.Fatalf("starting Dolt container: %v", err) + } + t.Cleanup(func() { + // Best effort: the dead-owned container is expected to be gone. + _ = testcontainers.TerminateContainer(ctr) + }) + return ctr.GetContainerID() + } + + // A sibling test binary's sweep (go test ./... runs them in parallel) + // removes a container as soon as it sees the owner dead, even mid-start. + // So the dead-owned container's owner is a child process that runs until + // the container is up and its volumes are read, and only then exits. + owner := exec.Command("sleep", "600") // outlives both starts + if err := owner.Start(); err != nil { + t.Fatalf("starting the owner process: %v", err) + } + endOwner := sync.OnceFunc(func() { + _ = owner.Process.Kill() + // Reaped, its pid reports ESRCH; a zombie would still read as alive. + _ = owner.Wait() + }) + t.Cleanup(endOwner) + + deadOwned := start(t, owner.Process.Pid) + liveOwned := start(t, os.Getpid()) + deadVolumes := containerVolumes(t, deadOwned) + if len(deadVolumes) == 0 { + t.Fatalf("dead-owned container %s has no volume; %s no longer declares VOLUME /var/lib/dolt?", deadOwned, DoltDockerImage) + } + endOwner() + + removed := sweepStaleDoltContainers() + + // Once its owner has exited, a sibling's sweep may remove the dead-owned + // container first, or still be removing it when ours fails to, so only + // its end state is checked, and given time to settle. + if slices.Contains(removed, liveOwned) { + t.Errorf("sweep removed the live-owned container %s", liveOwned) + } + if !dockerObjectGone("inspect", deadOwned) { + t.Errorf("dead-owned container %s still exists after the sweep", deadOwned) + } + for _, v := range deadVolumes { + if !dockerObjectGone("volume", "inspect", v) { + t.Errorf("volume %s of the dead-owned container survived the sweep", v) + } + } + if err := exec.Command("docker", "inspect", liveOwned).Run(); err != nil { + t.Errorf("live-owned container %s missing after the sweep: %v", liveOwned, err) + } +} + +// dockerObjectGone reports whether `docker args...`, an inspect of one object, +// starts failing within sweepTimeout, which also bounds a sibling's removal. +func dockerObjectGone(args ...string) bool { + deadline := time.Now().Add(sweepTimeout) + for exec.Command("docker", args...).Run() == nil { + if time.Now().After(deadline) { + return false + } + time.Sleep(100 * time.Millisecond) + } + return true +} + +// containerVolumes returns the names of the volumes mounted into container id. +func containerVolumes(t *testing.T, id string) []string { + t.Helper() + out, err := exec.Command("docker", "inspect", "--format", + `{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}`, id).Output() + if err != nil { + t.Fatalf("docker inspect %s: %v", id, err) + } + return strings.Fields(string(out)) +} diff --git a/internal/testutil/testdoltserver.go b/internal/testutil/testdoltserver.go index ad7ca817e..771ddb1c9 100644 --- a/internal/testutil/testdoltserver.go +++ b/internal/testutil/testdoltserver.go @@ -180,6 +180,7 @@ func startDoltContainer() error { ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() + reapStaleDoltContainers() ctr, err := dolt.Run(ctx, DoltDockerImage, dolt.WithDatabase("beads_test"), // Docker port-forwarding makes connections appear as non-localhost @@ -187,6 +188,7 @@ func startDoltContainer() error { // "localhost", so root@localhost won't match external connections. // Set to "%" so root can connect from any host. testcontainers.WithEnv(map[string]string{"DOLT_ROOT_HOST": "%"}), + ownerLabels(), ) if err != nil { return fmt.Errorf("starting Dolt container: %w", err) @@ -434,9 +436,11 @@ func startIsolatedDoltContainer(t *testing.T) *IsolatedDoltContainer { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() + reapStaleDoltContainers() ctr, err := dolt.Run(ctx, DoltDockerImage, dolt.WithDatabase("beads_test"), testcontainers.WithEnv(map[string]string{"DOLT_ROOT_HOST": "%"}), + ownerLabels(), ) if err != nil { t.Fatalf("starting Dolt container: %v", err) From f53010a23b0a169473b7da7a364d3dc60014ea79 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 02:50:26 +0000 Subject: [PATCH 030/151] ci(bazel): remote repo contents cache: main-push seed, lane readers, nightly verify (#7453) Port gascity's remote repo contents cache wiring (gascity #7467, #7478) to bazel.yml, gated on the repository variable RBE_REPO_CONTENTS_CACHE (unset/off: nothing; seed: rrc-seed on push to main; canary: + bazel-test reads; on: + every test lane reads). Mode remote only. - rbe job: an rrc step turns the variable into the rrc output. - Readers: the eight test lanes append startup --experimental_remote_repo_contents_cache and common --loading_phase_threads=64 to .bazelrc.local before Set up Bazel. - rrc-seed (push to main, id-token: write): a 30-minute rbe-rrc-writer-beads certificate from rbe-west's mint (.github/scripts/rrc-writer-credential.sh), then every lane's command (.github/scripts/rrc-lane-commands.txt) with --nobuild, --remote_cache only and --remote_upload_local_results (.github/scripts/rrc-run-lanes.sh). - rrc-verify (schedule/dispatch, issues: write): cold fetch of every lane and tools/bazel/rrc_verify.py against the cache; a mismatch opens an rrc-verify issue. - pr.yml, nightly.yml and bazel-farm.yml grant id-token: write and issues: write on their bazel.yml call (startup_failure otherwise); TestBazelWorkflowCallersGrantEveryRequestedPermission guards it. rrc-writer-credential.sh, rrc_verify.py and rrc_verify_test.py are byte copies of gascity's, pinned by sha256. Refs ga-vnycm2.29. Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/rrc-lane-commands.txt | 15 + .github/scripts/rrc-run-lanes.sh | 51 ++ .github/scripts/rrc-writer-credential.sh | 113 ++++ .github/workflows/bazel-farm.yml | 10 + .github/workflows/bazel.yml | 246 +++++++ .github/workflows/nightly.yml | 12 + .github/workflows/pr.yml | 10 + engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 32 + scripts/BUILD.bazel | 1 + scripts/bazel_cache_mode_test.go | 4 +- scripts/bazel_farm_workflow_test.go | 49 +- scripts/bazel_rrc_test.go | 783 +++++++++++++++++++++++ scripts/ci_analytics_workflow_test.go | 2 + scripts/ci_blacksmith_runner_test.go | 4 +- scripts/ci_workflow_test.go | 37 +- scripts/rbe_worker_env_test.go | 15 +- scripts/tracked_files_test.go | 13 +- tools/bazel/rrc_verify.py | 620 ++++++++++++++++++ tools/bazel/rrc_verify_test.py | 323 ++++++++++ 19 files changed, 2310 insertions(+), 30 deletions(-) create mode 100644 .github/scripts/rrc-lane-commands.txt create mode 100755 .github/scripts/rrc-run-lanes.sh create mode 100755 .github/scripts/rrc-writer-credential.sh create mode 100644 scripts/bazel_rrc_test.go create mode 100644 tools/bazel/rrc_verify.py create mode 100644 tools/bazel/rrc_verify_test.py diff --git a/.github/scripts/rrc-lane-commands.txt b/.github/scripts/rrc-lane-commands.txt new file mode 100644 index 000000000..08bd7eb5e --- /dev/null +++ b/.github/scripts/rrc-lane-commands.txt @@ -0,0 +1,15 @@ +# Every bazel.yml lane's loading and analysis, for the remote repo contents +# cache: rrc-seed runs each line with --nobuild on a push to main (uploading +# the repositories it fetches) and rrc-verify runs each cold every night. +# One `bazel` command per line (no shell quoting); # starts a comment. +# scripts/bazel_rrc_test.go checks that every --config a reading lane's +# bazel command uses appears here. +test //... --config=ci +test //... --config=embedded +test //... --config=integration +test //... --config=doltserver +test //... --config=doltserver-proxied +test //... --config=doltserver-integration +test //... --config=doltserver-cmd +build --config=pure //cmd/bd:bd //cmd/bd:bd_test //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test +build --config=js-wasm //internal/hooks:hooks_test diff --git a/.github/scripts/rrc-run-lanes.sh b/.github/scripts/rrc-run-lanes.sh new file mode 100755 index 000000000..d5f329a84 --- /dev/null +++ b/.github/scripts/rrc-run-lanes.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Loading and analysis of every bazel.yml lane, for the remote repo contents +# cache (bazel.yml's rrc-seed and rrc-verify jobs). +# +# Usage: rrc-run-lanes.sh [STARTUP_FLAG...] -- [COMMAND_FLAG...] +# +# Runs `bazel STARTUP_FLAG... --nobuild COMMAND_FLAG...` for each +# command line in .github/scripts/rrc-lane-commands.txt (or RRC_LANE_COMMANDS), +# in order, stopping at the first failure. --nobuild: repository rules, +# loading and analysis only, never a build or test action. +# +# Bazel exits 1 after `test --nobuild` even when loading and analysis succeed +# ("Couldn't start the build. Unable to run tests"). A command passes when it +# exits 0, or exits non-zero having printed "Build completed successfully" +# and no ERROR line but that one. +set -euo pipefail + +startup=() +while [ $# -gt 0 ] && [ "$1" != -- ]; do + startup+=("$1") + shift +done +[ $# -gt 0 ] || { + echo "usage: rrc-run-lanes.sh [STARTUP_FLAG...] -- [COMMAND_FLAG...]" >&2 + exit 2 +} +shift +flags=("$@") + +file=${RRC_LANE_COMMANDS:-$(dirname "${BASH_SOURCE[0]}")/rrc-lane-commands.txt} +mapfile -t cmds < <(sed -e 's/#.*//' -e 's/[[:space:]]*$//' "$file" | grep -v '^[[:space:]]*$' || true) +if [ "${#cmds[@]}" -eq 0 ]; then + echo "::error::no lane commands in $file" + exit 1 +fi + +log=$(mktemp) +trap 'rm -f "$log"' EXIT +summary=${GITHUB_STEP_SUMMARY:-/dev/null} +for cmd in "${cmds[@]}"; do + read -r -a args <<<"$cmd" + start=$(date +%s) + rc=0 + bazel ${startup[@]+"${startup[@]}"} "${args[@]}" --nobuild ${flags[@]+"${flags[@]}"} &1 | tee "$log" || rc=$? + if [ "$rc" -ne 0 ] && grep -q '^INFO: Build completed successfully' "$log" && + ! grep '^ERROR:' "$log" | grep -qv "^ERROR: Couldn't start the build. Unable to run tests$"; then + rc=0 + fi + echo "rrc: \`bazel $cmd\`: exit $rc, $(($(date +%s) - start))s" | tee -a "$summary" + [ "$rc" -eq 0 ] || exit "$rc" +done diff --git a/.github/scripts/rrc-writer-credential.sh b/.github/scripts/rrc-writer-credential.sh new file mode 100755 index 000000000..a650c7ddc --- /dev/null +++ b/.github/scripts/rrc-writer-credential.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# A 30-minute rbe-west client certificate for the remote repo contents cache +# writer (bazel.yml's rrc-seed job; engdocs/design/bazel-remote-repo-contents-cache.md). +# +# The job trades its GitHub Actions OIDC token (audience rbe-rrc-writer) for a +# certificate CN=rbe-rrc-writer from rbe-west's mint. The mint signs only for +# this repository's bazel.yml on a push to main (repository, ref, event_name +# and job_workflow_ref claims); pull_request, fork and Dependabot runs cannot +# get one (no OIDC token, or a refs/pull/N/merge one). On rbe-west that +# certificate reaches only GetCapabilities, the CAS and an action cache front +# (rrc-gate) that admits repo-contents entries and refuses everything else, so +# it can never write an execution result or execute anything. +# +# The key is an EC P-256 key (PKCS#8: Bazel's Netty TLS refuses a SEC1 "EC +# PRIVATE KEY") generated here in BAZEL_CI_SECRET_DIR, outside the workspace; +# it never leaves the runner. The OIDC token is masked. Outputs (to +# $GITHUB_OUTPUT when set, else stdout): cert, key, endpoint, instance; none +# when the mint answers 503 or cannot be reached (seeding switched off on +# rbe-west: infra nativelink-cas/west/README.md "Client contract"), which +# exits 0 with a warning. +# +# Tests only: RBE_RRC_MINT (a stand-in mint) and RBE_RRC_ENDPOINT_RE. +set -euo pipefail +MINT=${RBE_RRC_MINT:-https://rbe-mint.ops.gascity.com:8444} +ENDPOINT_RE=${RBE_RRC_ENDPOINT_RE:-'^grpcs://rbe-west\.ops\.gascity\.com(:443)?$'} +AUDIENCE=rbe-rrc-writer +dir=${BAZEL_CI_SECRET_DIR:?BAZEL_CI_SECRET_DIR is required} +out=${GITHUB_OUTPUT:-/dev/stdout} + +if [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || [ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ]; then + echo "::error title=rrc writer::no OIDC token request URL; the job needs permissions: id-token: write" >&2 + exit 1 +fi + +install -d -m 0700 "$dir" +umask 077 + +oidc=$(curl -sS --fail --connect-timeout 5 --max-time 30 --retry 3 --retry-all-errors \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$AUDIENCE" | jq -r '.value // empty') +if [ -z "$oidc" ]; then + echo "::error title=rrc writer::GitHub returned no OIDC token" >&2 + exit 1 +fi +printf '::add-mask::%s\n' "$oidc" + +openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$dir/rrc-writer.key" 2>/dev/null +csr=$(openssl req -new -key "$dir/rrc-writer.key" -subj "/CN=rbe-rrc-writer-request") +body=$(jq -cn --arg csr "$csr" '{csr_pem: $csr}') + +reply="$dir/rrc-mint.json" +code=000 +for attempt in 1 2 3 4; do + # --connect-timeout: a closed gate drops the SYN; fail fast, not in 60 s. + code=$(curl -sS -o "$reply" -w '%{http_code}' --connect-timeout 5 --max-time 60 \ + -H "Authorization: Bearer $oidc" -H 'content-type: application/json' \ + --data "$body" "$MINT/v1/rrc-writer/cert") || code=000 + # Retry what may pass on its own: a rate limit (429), the mint could not + # reach GitHub's JWKS (502), or the network (000). Everything else is an + # answer. + case "$code" in 429 | 502 | 000) ;; *) break ;; esac + [ "$attempt" -eq 4 ] || sleep $((attempt * 10)) +done +case "$code" in +200) ;; +503 | 000) + # The mint's switch is off, its CA is not installed, or rbe-west's fork + # edge is closed: seeding is off, not broken. No outputs, so the seed + # step skips; the push to main never fails on it. + echo "::warning title=rrc writer: seeding off (HTTP $code)::$(jq -r '.error // empty' "$reply" 2>/dev/null || true)" + exit 0 + ;; +*) + echo "::error title=rrc writer mint refused (HTTP $code)::$(jq -r '.error // empty' "$reply" 2>/dev/null || true)" + exit 1 + ;; +esac + +endpoint=$(jq -r '.endpoint // empty' "$reply") +instance=$(jq -r '.instance // empty' "$reply") +[[ $endpoint =~ $ENDPOINT_RE ]] || { + echo "::error::rrc writer mint returned endpoint '$endpoint'" >&2 + exit 1 +} +[ "$instance" = oss ] || { + echo "::error::rrc writer mint returned instance '$instance', want oss" >&2 + exit 1 +} +jq -r '.cert_pem // empty' "$reply" >"$dir/rrc-writer.crt" +subject=$(openssl x509 -in "$dir/rrc-writer.crt" -noout -subject -nameopt RFC2253) || { + echo "::error::rrc writer mint returned no certificate" >&2 + exit 1 +} +rdns=$(tr ',' '\n' <<<"${subject#subject=}") +cn=$(sed -n 's/^CN=//p' <<<"$rdns") +org=$(sed -n 's/^O=//p' <<<"$rdns") +if ! [[ $cn =~ ^rbe-rrc-writer(-[a-z0-9]+)?$ ]] || [ "$org" != gascity ]; then + echo "::error::rrc writer mint returned a certificate for '$subject'" >&2 + exit 1 +fi +# The certificate must belong to the key generated above. +if [ "$(openssl x509 -in "$dir/rrc-writer.crt" -noout -pubkey | openssl sha256)" != "$(openssl pkey -in "$dir/rrc-writer.key" -pubout | openssl sha256)" ]; then + echo "::error::rrc writer mint returned a certificate for another key" >&2 + exit 1 +fi +openssl x509 -in "$dir/rrc-writer.crt" -noout -subject -enddate +rm -f "$reply" +{ + echo "cert=$dir/rrc-writer.crt" + echo "key=$dir/rrc-writer.key" + echo "endpoint=$endpoint" + echo "instance=$instance" +} >>"$out" diff --git a/.github/workflows/bazel-farm.yml b/.github/workflows/bazel-farm.yml index 7ab2c0a7b..caeb0ad2f 100644 --- a/.github/workflows/bazel-farm.yml +++ b/.github/workflows/bazel-farm.yml @@ -120,8 +120,18 @@ jobs: name: Bazel Farm needs: authorize if: ${{ needs.authorize.outputs.allowed == 'true' }} + # A called workflow's job gets at most this call's token permissions, and + # GitHub checks them when the run starts, even for a job whose if: skips + # it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks + # id-token: write (push to main only) and rrc-verify issues: write + # (schedule and dispatch only). Neither runs for this event, every other + # bazel.yml job keeps contents: read, and rbe-west's mint refuses any OIDC + # token but a main-push bazel.yml one + # (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission). permissions: contents: read + id-token: write + issues: write uses: ./.github/workflows/bazel.yml # Exactly the four RBE secrets: never the rbe-prewarm app private key # (B1, security review of bdef342d5). This pull_request_target run is diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index eeb98a410..ce9bf223b 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -424,6 +424,7 @@ jobs: enabled: ${{ steps.decide.outputs.enabled }} mode: ${{ steps.decide.outputs.mode }} tier: ${{ steps.decide.outputs.tier }} + rrc: ${{ steps.rrc.outputs.rrc }} steps: # The expressions' == is case-insensitive (RBE_WEST_WORKERS=True counts). # A push, merge_group or dispatch has no pull_request, so FORK is false. @@ -496,6 +497,37 @@ jobs: } >> "$GITHUB_OUTPUT" echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, dependabot: $DEPENDABOT, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + # Bazel's remote repo contents cache on rbe-west (ga-vnycm2.29, ported + # from gascity's bazel.yml; gascity + # engdocs/design/bazel-remote-repo-contents-cache.md): extracted + # external repositories (go_deps, the Go SDK, the LLVM slice) come from + # rbe-west's action cache instead of being re-fetched by every cold lane + # client. vars.RBE_REPO_CONTENTS_CACHE rolls it out: + # unset/off nothing (the default); + # seed main pushes seed the cache (rrc-seed); no lane reads it; + # canary seed, and bazel-test reads; + # on seed, and every lane in .github/scripts/rrc-lane-commands.txt + # reads. + # Mode remote only: fork pull_request runs see no vars, and only + # rbe-west's trusted edge serves the entries. Any other value is off, + # with a warning, so a typo never fails the required gate. Like decide, + # this step runs no repository code (it comes before the checkout). + - name: Remote repo contents cache + id: rrc + env: + RRC: ${{ vars.RBE_REPO_CONTENTS_CACHE }} + MODE: ${{ steps.decide.outputs.mode }} + run: | + rrc=off + case "$RRC" in + "" | off) ;; + seed | canary | on) + if [ "$MODE" = remote ]; then rrc=$RRC; fi ;; + *) echo "::warning title=RBE_REPO_CONTENTS_CACHE::unknown value '$RRC' (want off, seed, canary or on); treated as off" ;; + esac + echo "rrc=$rrc" >> "$GITHUB_OUTPUT" + echo "Remote repo contents cache: **$rrc**" >> "$GITHUB_STEP_SUMMARY" + # The worker-env preflight (see the job's comment): the PR's own copy, # so a PR that syncs it passes. Only the two paths it compares. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -785,6 +817,22 @@ jobs: - name: Verify restored Go modules run: go mod verify + # The remote repo contents cache, read-only (the rbe job's rrc output: + # canary, bazel-test; on, every lane). Each cached repository costs + # several sequential round trips, resolved on Skyframe's loading + # threads (default: the CPU count), so without 64 of them a cached read + # is slower than fetching. Both lines are key neutral (startup and + # loading only). Reading only: setup-bazel's rc keeps + # --noremote_upload_local_results, so nothing here writes; rrc-seed + # writes, on push to main. Before Set up Bazel: no server starts first. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && (needs.rbe.outputs.rrc == 'on' || needs.rbe.outputs.rrc == 'canary') }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1045,6 +1093,15 @@ jobs: with: node-version: '24' + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1333,6 +1390,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1505,6 +1571,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1650,6 +1725,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1770,6 +1854,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1922,6 +2015,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -2085,6 +2187,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -2432,3 +2543,138 @@ jobs: env: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # The remote repo contents cache's only writer (ga-vnycm2.29, ported from + # gascity's bazel.yml). On a push to main, with the rbe job's rrc not off, + # run every lane's loading and analysis (.github/scripts/rrc-lane-commands.txt, + # --nobuild): only main's repository rules and Starlark run (gazelle's + # fetch_repo, the Go SDK and LLVM downloads, extraction: what every lane + # already runs on its runner), never a build or test action. Bazel uploads + # each reproducible repository's tree (--remote_upload_local_results) with + # a 30-minute rbe-rrc-writer-beads certificate that rbe-west's mint signs + # only for this workflow on a push to main + # (.github/scripts/rrc-writer-credential.sh); rbe-west's rrc-gate admits + # repo-contents entries from it and refuses every other action cache + # write, and the certificate cannot Execute. Cache only (--remote_cache, + # no executor). Not a lane: no result output, never in pr.yml's gate + # (which never sees it anyway: push only), so a failed seed only leaves the + # lanes fetching. setup-bazel runs in local mode (no secrets): its rc, the + # runner cache and the pinned Bazel, nothing else. The only bazel.yml job + # with id-token: write; every caller must grant it (GitHub checks a called + # workflow's job permissions at startup, even for a job that skips), and + # the mint refuses any token but a main-push bazel.yml one. + rrc-seed: + name: repo contents cache seed + needs: rbe + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + id-token: write # the rbe-rrc-writer-beads certificate (GitHub OIDC) + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Bazel + uses: ./.github/actions/setup-bazel + - name: rbe-rrc-writer certificate + id: writer + env: + BAZEL_CI_SECRET_DIR: ${{ runner.temp }}/bazel-ci-secret + run: bash .github/scripts/rrc-writer-credential.sh + - name: Seed the remote repo contents cache + if: ${{ steps.writer.outputs.cert != '' }} + env: + CERT: ${{ steps.writer.outputs.cert }} + KEY: ${{ steps.writer.outputs.key }} + ENDPOINT: ${{ steps.writer.outputs.endpoint }} + INSTANCE: ${{ steps.writer.outputs.instance }} + run: >- + bash .github/scripts/rrc-run-lanes.sh --experimental_remote_repo_contents_cache -- + --loading_phase_threads=64 + "--remote_cache=$ENDPOINT" "--remote_instance_name=$INSTANCE" + "--tls_client_certificate=$CERT" "--tls_client_key=$KEY" + --remote_upload_local_results + - name: Remove the writer key + if: ${{ always() }} + run: rm -f -- "$RUNNER_TEMP/bazel-ci-secret/rrc-writer.key" "$RUNNER_TEMP/bazel-ci-secret/rrc-writer.crt" + + # Nightly check of the remote repo contents cache (gascity design R1/R2). A + # cached repository tree is trusted by every reader, and nothing on the + # reader checks it against what its rule produces. So, from nightly.yml (or + # a dispatch), fetch every lane's repositories cold (rrc-lane-commands.txt + # with --nobuild and no repo contents cache, Bazel set up exactly as a + # lane) and compare each repository's tree and marker with the cached + # entry a lane would read for the same key (tools/bazel/rrc_verify.py, + # reads only, with the lanes' certificate and the endpoint from + # setup-bazel's rc: no secret is referenced outside setup-bazel). A + # mismatch is a poisoned entry or a rule falsely marked reproducible: the + # job fails and opens (or comments on) an rrc-verify issue. Not a lane: + # never gated. The only bazel.yml job with issues: write; every caller + # must grant it, as for rrc-seed's id-token. + rrc-verify: + name: repo contents cache verify + needs: rbe + if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + permissions: + contents: read + issues: write # the rrc-verify alert issue + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} + - name: Cold fetch (every lane, no repo contents cache) + run: bash .github/scripts/rrc-run-lanes.sh -- --loading_phase_threads=64 + - name: Compare with the cache + id: verify + run: | + set -euo pipefail + secret_dir="$RUNNER_TEMP/bazel-ci-secret" + prefix='build:remote-exec --remote_executor=' + endpoint=$(grep -F -m1 "$prefix" "$secret_dir/ci.bazelrc" || true) + endpoint=${endpoint#"$prefix"} + [ -n "$endpoint" ] || { echo "::error::setup-bazel's rc names no executor"; exit 1; } + python3 tools/bazel/rrc_verify.py \ + --output-base "$(bazel info output_base)" \ + --endpoint "$endpoint" --instance oss \ + --cert "$secret_dir/client.crt" --key "$secret_dir/client.key" \ + --bazel-version "$(tr -d '[:space:]' < .bazelversion)" \ + --json "$RUNNER_TEMP/rrc-verify.json" --summary "$GITHUB_STEP_SUMMARY" + - name: Upload the report + if: ${{ always() && hashFiles(format('{0}/rrc-verify.json', runner.temp)) != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: rrc-verify-${{ github.run_attempt }} + path: ${{ runner.temp }}/rrc-verify.json + retention-days: 30 + - name: Alert (rrc-verify issue) + if: ${{ failure() && steps.verify.outcome == 'failure' }} + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + title="rrc-verify: remote repo contents cache differs from a cold fetch" + body="The nightly rrc-verify run found cached repository trees that differ from a cold fetch, or could not check them: $RUN_URL (summary and the rrc-verify report artifact). + + A mismatch is a poisoned entry or a repository rule falsely marked reproducible (gascity engdocs/design/bazel-remote-repo-contents-cache.md, R1/R2). Stop readers first: set the repository variable RBE_REPO_CONTENTS_CACHE to off. Then follow gascity/infra nativelink-cas/west/README.md (remote repo contents cache writer, poisoned entries)." + gh label create rrc-verify -R "$GITHUB_REPOSITORY" --color B60205 \ + --description "Remote repo contents cache verification failures" 2>/dev/null || true + n=$(gh issue list -R "$GITHUB_REPOSITORY" --label rrc-verify --state open --json number --jq '.[0].number // empty') + if [ -n "$n" ]; then + gh issue comment "$n" -R "$GITHUB_REPOSITORY" --body "Again: $RUN_URL" + else + gh issue create -R "$GITHUB_REPOSITORY" --label rrc-verify --title "$title" --body "$body" + fi diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 56f6a23e7..7bd190a2d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -25,6 +25,18 @@ jobs: # engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md). bazel: name: Bazel (fresh, full detail) + # A called workflow's job gets at most this call's token permissions, and + # GitHub checks them when the run starts, even for a job whose if: skips + # it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks + # id-token: write (push to main only) and rrc-verify issues: write + # (schedule and dispatch only). rrc-verify runs here (its alert issue); + # rrc-seed never does, every other bazel.yml job keeps contents: read, + # and rbe-west's mint refuses any OIDC token but a main-push bazel.yml one + # (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission). + permissions: + contents: read + id-token: write + issues: write uses: ./.github/workflows/bazel.yml secrets: RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 9483e2b36..22dc8c23e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1347,8 +1347,18 @@ jobs: # build their own bd via this call's rbe job. bazel: name: Bazel + # A called workflow's job gets at most this call's token permissions, and + # GitHub checks them when the run starts, even for a job whose if: skips + # it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks + # id-token: write (push to main only) and rrc-verify issues: write + # (schedule and dispatch only). Neither runs for this event, every other + # bazel.yml job keeps contents: read, and rbe-west's mint refuses any OIDC + # token but a main-push bazel.yml one + # (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission). permissions: contents: read + id-token: write + issues: write uses: ./.github/workflows/bazel.yml secrets: RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index c0c340b6d..ce91016b0 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -620,6 +620,38 @@ job's `if:` or rotating the key, in either of two ways: step's own kill switch, checked before any network call. Deleting the variable does not disable pre-warming; it restores the default of 1. +## Remote Repo Contents Cache + +A cold lane client re-runs its repository rules (gazelle's `go_deps`, the Go +SDK, the LLVM slice) before its first action. Bazel 9's remote repo contents +cache serves those extracted trees from rbe-west's `oss` action cache instead +(ported from gascity's `bazel.yml`; design: gascity +`engdocs/design/bazel-remote-repo-contents-cache.md`). The repository variable +`RBE_REPO_CONTENTS_CACHE` rolls it out, in mode `remote` only: + +| value | effect | +|---|---| +| unset / `off` | nothing (default) | +| `seed` | each push to main runs `bazel.yml`'s `rrc-seed` job: every lane's command in `.github/scripts/rrc-lane-commands.txt` with `--nobuild`, uploading repository trees with a 30-minute `rbe-rrc-writer-beads` certificate that rbe-west's mint signs only for that job (GitHub OIDC; `.github/scripts/rrc-writer-credential.sh`). rbe-west's `rrc-gate` admits only repo-contents entries from it | +| `canary` | `seed`, and the `test` lane reads the cache | +| `on` | `seed`, and every lane but the release cross-compile and the package gates reads it | + +Readers append `startup --experimental_remote_repo_contents_cache` and +`common --loading_phase_threads=64` to `.bazelrc.local` before setting up +Bazel; both are key neutral, and the lanes keep +`--noremote_upload_local_results`. While the variable is not `off`, +nightly.yml's call also runs `rrc-verify`: a cold fetch of every lane, +compared with the cached entries by `tools/bazel/rrc_verify.py`; a mismatch +fails the job and opens an `rrc-verify` issue. + +`rrc-seed` asks for `id-token: write` and `rrc-verify` for `issues: write`, so +every caller of `bazel.yml` (pr.yml, nightly.yml, bazel-farm.yml) grants both: +GitHub checks a called workflow's job permissions when the run starts, even +for jobs that skip. Neither job runs on a pull_request, merge_group or +pull_request_target event, and every other `bazel.yml` job keeps +`contents: read`. Rollback: set the variable to `off` (or `seed`, to keep the +cache warm without readers). + ## Required Check Contract After the aggregate checks are verified on the branch, branch protection or the diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index a91b79346..08cf03e48 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -41,6 +41,7 @@ go_test( "bazel_hermetic_cc_test.go", "bazel_policy_test.go", "bazel_rbe_prewarm_test.go", + "bazel_rrc_test.go", "ci_actions_policy_test.go", "ci_analytics_workflow_test.go", "ci_blacksmith_runner_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 7ee9e653e..8212d4efd 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -212,8 +212,8 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { var lanes []string for name, job := range workflow.Jobs { - if name == bazelRBEJobName { - continue + if name == bazelRBEJobName || isBazelRRCJob(name) { + continue // not lanes: never in a PR call (bazel_rrc_test.go) } runs := bazelLaneRunModes(t, name, job.If, s.with)[mode] switch { diff --git a/scripts/bazel_farm_workflow_test.go b/scripts/bazel_farm_workflow_test.go index 36a40b5b0..887130463 100644 --- a/scripts/bazel_farm_workflow_test.go +++ b/scripts/bazel_farm_workflow_test.go @@ -94,7 +94,8 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { } // Permissions: contents: read at the workflow and on every job, and - // nothing else anywhere in the file. + // nothing else anywhere in the file, but the farm call's grant of what + // bazel.yml's rrc jobs ask for (bazelCallPermissions). readOnly := map[string]any{"contents": "read"} if !reflect.DeepEqual(doc.Permissions, readOnly) { t.Errorf("workflow permissions = %v, want %v", doc.Permissions, readOnly) @@ -103,8 +104,12 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { var jobs []string for name, job := range workflow.Jobs { jobs = append(jobs, name) - if !reflect.DeepEqual(job.Permissions, readOnly) { - t.Errorf("job %s permissions = %v, want %v", name, job.Permissions, readOnly) + want := readOnly + if name == "farm" { + want = bazelCallPermissions + } + if !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("job %s permissions = %v, want %v", name, job.Permissions, want) } if len(job.Env) != 0 { t.Errorf("job %s env = %v; want none", name, job.Env) @@ -116,7 +121,8 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { } secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { - if strings.Contains(path, "permissions") && !key && value != "read" { + if strings.Contains(path, "permissions") && !key && value != "read" && + !(strings.HasPrefix(path, ".jobs.farm.permissions.") && bazelCallPermissions[strings.TrimPrefix(path, ".jobs.farm.permissions.")] == value) { t.Errorf("%s = %q; only read permissions are allowed", path, value) } if key && value == "continue-on-error" { @@ -227,6 +233,19 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { } } +// bazelCallPermissions: every bazel.yml call's token permissions (farm, +// pr.yml, nightly.yml). A called workflow's job gets at most its caller's, +// and GitHub checks them when the run starts, even for a job whose if: +// skips it, so each call grants what bazel.yml's rrc jobs ask for: +// rrc-seed's id-token: write (push to main only) and rrc-verify's issues: +// write (schedule and dispatch only). Neither runs on pull_request, +// merge_group or pull_request_target, and every other bazel.yml job, every +// lane that runs PR or fork code included, keeps contents: read +// (TestBazelWorkflowPermissionsReadOnly), so no PR-authored step ever holds +// either; rbe-west's mint refuses any OIDC token but a main-push bazel.yml +// one anyway. +var bazelCallPermissions = map[string]any{"contents": "read", "id-token": "write", "issues": "write"} + // bazel.yml's side of the farm: the pinned checkout, the one decision input, // and nothing a pull_request_target run could hand a privileged consumer. func TestBazelWorkflowForkFarmInputs(t *testing.T) { @@ -246,6 +265,15 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) { if actionFamily(step.Uses) != "actions/checkout" { continue } + // The rrc jobs run on push to main, schedule and dispatch only, + // never on pull_request_target: they check out the event's ref, + // with no fork opt-in. + if isBazelRRCJob(name) { + if want := map[string]string{"persist-credentials": "false"}; !reflect.DeepEqual(step.With, want) { + t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want) + } + continue + } checkouts++ want := map[string]string{"ref": bazelCheckoutRef, "persist-credentials": "false", "allow-unsafe-pr-checkout": bazelAllowUnsafeCheckout} // F3: package-mcp/package-npm's detect step diffs PR_BASE_SHA @@ -271,8 +299,8 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) { // shared gascity credential). rbe's checkout (sparse) comes after its // decide step, for the worker-env preflight // (TestBazelRBEWorkerEnvPreflight). - if checkouts != len(workflow.Jobs)-1 { - t.Errorf("%d checkouts in %s, want one per job excluding %s (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, len(workflow.Jobs)-1) + if want := len(workflow.Jobs) - 1 - len(bazelRRCJobs); checkouts != want { + t.Errorf("%d checkouts in %s, want one per job excluding %s and the rrc jobs (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, want) } if got := workflow.job(t, bazelRBEJobName).Steps[0].Env["FORK_FARM"]; got != bazelForkFarmValue { t.Errorf("rbe FORK_FARM = %q, want %q", got, bazelForkFarmValue) @@ -617,7 +645,11 @@ func TestBazelFarmCacheModeStaysReadOnly(t *testing.T) { } // bazel.yml's token: exactly contents: read, at the top and on any job that -// declares permissions (a call cannot exceed its caller's, but pin it). +// declares permissions (a call cannot exceed its caller's, but pin it), +// except the rrc jobs: rrc-seed's id-token: write and rrc-verify's issues: +// write (pinned by TestBazelRRCSeedJob and TestBazelRRCVerifyJob; neither +// runs on a pull_request, merge_group or pull_request_target event, so the +// callers' wider grants reach no lane and no fork-authored code). func TestBazelWorkflowPermissionsReadOnly(t *testing.T) { var doc struct { Permissions any `yaml:"permissions"` @@ -630,6 +662,9 @@ func TestBazelWorkflowPermissionsReadOnly(t *testing.T) { t.Errorf("%s permissions = %v, want exactly %v", bazelWorkflowName, doc.Permissions, readOnly) } for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if isBazelRRCJob(name) { + continue + } if job.Permissions != nil && !reflect.DeepEqual(job.Permissions, readOnly) { t.Errorf("%s job %s permissions = %v, want none or exactly %v", bazelWorkflowName, name, job.Permissions, readOnly) } diff --git a/scripts/bazel_rrc_test.go b/scripts/bazel_rrc_test.go new file mode 100644 index 000000000..f423b7337 --- /dev/null +++ b/scripts/bazel_rrc_test.go @@ -0,0 +1,783 @@ +package scripts_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "slices" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// Bazel's remote repo contents cache on rbe-west (ga-vnycm2.29, ported from +// gascity's bazel.yml, gascity #7467 and #7478). bazel.yml's rbe job turns +// vars.RBE_REPO_CONTENTS_CACHE into its rrc output (off, seed, canary, on); +// each reading lane's reader step appends bazelRRCReadLines to +// .bazelrc.local; the rrc-seed job, on push to main only, runs every lane's +// loading and analysis (.github/scripts/rrc-lane-commands.txt, through +// rrc-run-lanes.sh) and uploads repository trees with a 30-minute +// rbe-rrc-writer-beads certificate (rrc-writer-credential.sh); the +// rrc-verify job, nightly, compares the cache with a cold fetch +// (tools/bazel/rrc_verify.py). + +const ( + bazelRRCReadStep = "Remote repo contents cache (read)" + bazelRRCModeStepID = "rrc" + bazelRRCVar = "vars.RBE_REPO_CONTENTS_CACHE" + bazelRRCSeedJob = "rrc-seed" + bazelRRCSeedJobIf = "${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }}" + bazelRRCVerifyJob = "rrc-verify" + bazelRRCVerifyJobIf = "${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }}" + bazelRRCCanaryLane = "bazel-test" + bazelRRCCredential = ".github/scripts/rrc-writer-credential.sh" + bazelRRCRunLanes = ".github/scripts/rrc-run-lanes.sh" + bazelRRCLaneCmds = ".github/scripts/rrc-lane-commands.txt" + bazelRRCVerifyTool = "tools/bazel/rrc_verify.py" + bazelRRCVerifyTest = "tools/bazel/rrc_verify_test.py" + bazelSetupBazelUses = "./.github/actions/setup-bazel" +) + +// bazelRRCReadLanes: the lanes that read the cache under `on` (bazel-test +// alone under `canary`). Not bazel-release-cross (other target platforms, +// not seeded) nor the package gates. +var bazelRRCReadLanes = []string{ + "bazel-cmd-dolt", "bazel-doltserver", "bazel-embedded", "bazel-integration", + "bazel-proxied", "bazel-pure", "bazel-server-storage", "bazel-test", +} + +// bazelRRCReadLines: the reader's .bazelrc.local lines; both are key neutral +// (a startup option, and loading parallelism). +var bazelRRCReadLines = []string{ + "startup --experimental_remote_repo_contents_cache", + "common --loading_phase_threads=64", +} + +// bazelRRCJobs: bazel.yml's jobs that are not lanes (no result output, never +// gated): the remote repo contents cache's writer and its nightly check. +var bazelRRCJobs = []string{bazelRRCSeedJob, bazelRRCVerifyJob} + +// isBazelRRCJob reports whether a bazel.yml job is one of bazelRRCJobs. +func isBazelRRCJob(name string) bool { return slices.Contains(bazelRRCJobs, name) } + +// evalRRCIf evaluates a job or step if: with ctx, failing the test on an +// expression the evaluator cannot read. +func evalRRCIf(t *testing.T, expr string, ctx map[string]string) bool { + t.Helper() + v, err := evalGHExpr(expr, ctx) + if err != nil { + t.Fatalf("evalGHExpr(%q): %v", expr, err) + } + return ghTruthy(v) +} + +// runRRCScript runs a step's script (or a script file) under GitHub's bash +// flags in dir with env, returning its combined output. +func runRRCScript(t *testing.T, dir, script string, env map[string]string) (string, error) { + t.Helper() + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script) + cmd.Dir = dir + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir} + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + out, err := cmd.CombinedOutput() + return string(out), err +} + +// rrcOutputs reads a $GITHUB_OUTPUT file (key=value lines). +func rrcOutputs(t *testing.T, path string) map[string]string { + t.Helper() + out := map[string]string{} + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return out + } + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if k, v, ok := strings.Cut(line, "="); ok { + out[k] = v + } + } + return out +} + +func rrcStep(t *testing.T, job ciWorkflowJob, what string, pick func(ciWorkflowStep) bool) ciWorkflowStep { + t.Helper() + var found []ciWorkflowStep + for _, s := range job.Steps { + if pick(s) { + found = append(found, s) + } + } + if len(found) != 1 { + t.Fatalf("want one %s step, got %d", what, len(found)) + } + return found[0] +} + +// TestBazelRRCModeStep runs the rbe job's rrc step for every variable value +// and execution mode: only mode remote enables anything, unset and off are +// off, and an unknown value is off with a warning (a typo never fails the +// required gate). It reads no secret and runs before the checkout. +func TestBazelRRCModeStep(t *testing.T) { + rbe := readCIWorkflow(t, bazelWorkflowName).Jobs[bazelRBEJobName] + if got := rbe.Outputs["rrc"]; got != "${{ steps.rrc.outputs.rrc }}" { + t.Errorf("rbe job output rrc = %q", got) + } + step := rrcStep(t, rbe, "rrc", func(s ciWorkflowStep) bool { return s.ID == bazelRRCModeStepID }) + if want := map[string]string{"RRC": "${{ " + bazelRRCVar + " }}", "MODE": "${{ steps.decide.outputs.mode }}"}; !reflect.DeepEqual(step.Env, want) { + t.Errorf("rrc step env %v, want %v", step.Env, want) + } + if step.If != "" { + t.Errorf("rrc step if %q; it must always set the output", step.If) + } + for i, s := range rbe.Steps { + if s.Uses != "" { + t.Errorf("rbe step %d (%s) precedes nothing it should: the rrc step must come before any checkout", i, s.Uses) + } + if s.ID == bazelRRCModeStepID { + break + } + } + for _, mode := range []string{"remote", "fork-ro", "fork-rw", "cache", "local", "skip"} { + for value, want := range map[string]string{"": "off", "off": "off", "seed": "seed", "canary": "canary", "on": "on", "On": "off", "true": "off", "on;x": "off"} { + if mode != "remote" { + want = "off" + } + dir := t.TempDir() + output := filepath.Join(dir, "output") + out, err := runRRCScript(t, dir, step.Run, map[string]string{ + "RRC": value, "MODE": mode, "GITHUB_OUTPUT": output, "GITHUB_STEP_SUMMARY": filepath.Join(dir, "summary"), + }) + if err != nil { + t.Errorf("mode %s value %q: %v\n%s", mode, value, err, out) + continue + } + if got := rrcOutputs(t, output)["rrc"]; got != want { + t.Errorf("mode %s value %q: rrc=%q, want %q", mode, value, got, want) + } + unknown := !slices.Contains([]string{"", "off", "seed", "canary", "on"}, value) + if warned := strings.Contains(out, "::warning"); warned != unknown { + t.Errorf("mode %s value %q: warning %v, want %v:\n%s", mode, value, warned, unknown, out) + } + } + } +} + +// TestBazelRRCReadSteps: exactly bazelRRCReadLanes read, in mode remote only +// (fork and local runs never: only rbe-west's trusted edge serves the +// entries), bazel-test under canary and every one of them under on; each +// writes exactly bazelRRCReadLines, never mentions uploads, and comes before +// the job's Set up Bazel (so no Bazel server starts without it). +func TestBazelRRCReadSteps(t *testing.T) { + wf := readCIWorkflow(t, bazelWorkflowName) + var readers []string + for name, job := range wf.Jobs { + read, setup := -1, -1 + for i, s := range job.Steps { + switch { + case s.Name == bazelRRCReadStep: + if read >= 0 { + t.Errorf("%s has two %q steps", name, bazelRRCReadStep) + } + read = i + case s.Uses == bazelSetupBazelUses && setup < 0: + setup = i + } + } + if read < 0 { + continue + } + readers = append(readers, name) + if setup < 0 || read > setup { + t.Errorf("%s: %q at step %d, Set up Bazel at %d; the reader must come first", name, bazelRRCReadStep, read, setup) + } + step := job.Steps[read] + for _, mode := range []string{"remote", "fork-ro", "fork-rw", "cache", "local"} { + for _, rrc := range []string{"off", "seed", "canary", "on"} { + want := mode == "remote" && (rrc == "on" || (rrc == "canary" && name == bazelRRCCanaryLane)) + if got := evalRRCIf(t, step.If, map[string]string{"needs.rbe.outputs.mode": mode, "needs.rbe.outputs.rrc": rrc}); got != want { + t.Errorf("%s mode %s rrc %s: reader runs %v, want %v", name, mode, rrc, got, want) + } + } + } + dir := t.TempDir() + if out, err := runRRCScript(t, dir, step.Run, nil); err != nil { + t.Fatalf("%s reader: %v\n%s", name, err, out) + } + got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, ".bazelrc.local")), "\n") + if !slices.Equal(got, bazelRRCReadLines) { + t.Errorf("%s reader writes %q, want %q", name, got, bazelRRCReadLines) + } + if strings.Contains(step.Run, "upload") || len(step.Env) != 0 { + t.Errorf("%s reader mentions uploads or takes env; lanes only read", name) + } + } + sort.Strings(readers) + if !slices.Equal(readers, bazelRRCReadLanes) { + t.Errorf("lanes with a reader step %v, want %v", readers, bazelRRCReadLanes) + } +} + +// bazelRRCLaneCommands parses rrc-lane-commands.txt as rrc-run-lanes.sh does. +func bazelRRCLaneCommands(t *testing.T, root string) []string { + t.Helper() + var cmds []string + for _, line := range strings.Split(readPolicyFile(t, root, bazelRRCLaneCmds), "\n") { + line, _, _ = strings.Cut(line, "#") + if line = strings.TrimSpace(line); line != "" { + cmds = append(cmds, line) + } + } + return cmds +} + +// TestBazelRRCLaneCommandsCoverEveryReader: rrc-seed seeds (and rrc-verify +// checks) what the readers fetch, so every --config a reading lane passes to +// `bazel test` or `bazel build` has a command in rrc-lane-commands.txt, and +// every command there is a reading lane's. +func TestBazelRRCLaneCommandsCoverEveryReader(t *testing.T) { + root := sourceRepoRoot(t) + seeded := map[string]bool{} + for _, cmd := range bazelRRCLaneCommands(t, root) { + f := strings.Fields(cmd) + if f[0] != "test" && f[0] != "build" { + t.Errorf("lane command %q: want a test or build command", cmd) + } + for _, a := range f[1:] { + if c, ok := strings.CutPrefix(a, "--config="); ok { + seeded[c] = true + } + } + } + used := map[string]bool{} + invocation := regexp.MustCompile(`\bbazel (?:test|build)\b[^\n]*(?:\\\n[^\n]*)*`) + config := regexp.MustCompile(`--config=([a-z0-9-]+)`) + wf := readCIWorkflow(t, bazelWorkflowName) + for _, lane := range bazelRRCReadLanes { + for _, s := range wf.Jobs[lane].Steps { + for _, inv := range invocation.FindAllString(s.Run, -1) { + for _, m := range config.FindAllStringSubmatch(inv, -1) { + if m[1] != "sole-run" && m[1] != "fresh" { // test-result options, nothing fetched + used[m[1]] = true + } + } + } + } + } + if len(used) < len(bazelRRCReadLanes) { + t.Fatalf("found the configs %v in %d reading lanes; the scan is broken", used, len(bazelRRCReadLanes)) + } + for c := range used { + if !seeded[c] { + t.Errorf("a reading lane runs bazel with --config=%s, which %s does not seed", c, bazelRRCLaneCmds) + } + } + for c := range seeded { + if !used[c] { + t.Errorf("%s seeds --config=%s, which no reading lane uses", bazelRRCLaneCmds, c) + } + } +} + +// TestRRCRunLanes runs rrc-run-lanes.sh with a stub bazel: one --nobuild +// invocation per lane command, in order, startup flags first and command +// flags after; `test --nobuild`'s exit 1 after a successful analysis +// passes, a failed analysis fails, and an empty command list fails. +func TestRRCRunLanes(t *testing.T) { + root := sourceRepoRoot(t) + script := filepath.Join(root, bazelRRCRunLanes) + dir := t.TempDir() + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + calls := filepath.Join(dir, "calls") + stub := "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >>" + calls + "\n" + + "if [ -n \"${BAZEL_TEST_FAIL:-}\" ]; then echo 'ERROR: no such package'; echo 'ERROR: Build did NOT complete successfully'; exit 1; fi\n" + + "case \"$*\" in *' build '*) echo 'INFO: Build completed successfully, 0 total actions'; exit 0 ;; esac\n" + + "echo 'INFO: Build completed successfully, 0 total actions'\necho \"ERROR: Couldn't start the build. Unable to run tests\"\nexit 1\n" + if err := os.WriteFile(filepath.Join(bin, "bazel"), []byte(stub), 0o755); err != nil { + t.Fatal(err) + } + env := map[string]string{ + "PATH": bin + string(os.PathListSeparator) + os.Getenv("PATH"), + "GITHUB_STEP_SUMMARY": filepath.Join(dir, "summary"), + } + run := "bash " + script + " --experimental_remote_repo_contents_cache -- --loading_phase_threads=64 --remote_upload_local_results" + if out, err := runRRCScript(t, dir, run, env); err != nil { + t.Fatalf("rrc-run-lanes.sh: %v\n%s", err, out) + } + var want []string + for _, cmd := range bazelRRCLaneCommands(t, root) { + want = append(want, "--experimental_remote_repo_contents_cache "+cmd+" --nobuild --loading_phase_threads=64 --remote_upload_local_results") + } + if got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, "calls")), "\n"); !slices.Equal(got, want) { + t.Errorf("bazel calls:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } + if err := os.Remove(calls); err != nil { + t.Fatal(err) + } + if out, err := runRRCScript(t, dir, "bash "+script+" --", env); err != nil { + t.Fatalf("without flags: %v\n%s", err, out) + } + if got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, "calls")), "\n"); len(got) == 0 || !strings.HasPrefix(got[0], "test ") || !strings.HasSuffix(got[0], " --nobuild") { + t.Errorf("without flags, first call %q", got) + } + + failEnv := map[string]string{"BAZEL_TEST_FAIL": "1"} + for k, v := range env { + failEnv[k] = v + } + if out, err := runRRCScript(t, dir, run, failEnv); err == nil { + t.Errorf("a failed analysis passed:\n%s", out) + } + empty := filepath.Join(dir, "empty.txt") + if err := os.WriteFile(empty, []byte("# nothing\n\n"), 0o644); err != nil { + t.Fatal(err) + } + emptyEnv := map[string]string{"RRC_LANE_COMMANDS": empty} + for k, v := range env { + emptyEnv[k] = v + } + if out, err := runRRCScript(t, dir, run, emptyEnv); err == nil { + t.Errorf("no lane commands passed; it must fail rather than seed nothing:\n%s", out) + } + if out, err := runRRCScript(t, dir, "bash "+script+" --flag", env); err == nil { + t.Errorf("no -- separator passed:\n%s", out) + } +} + +// TestBazelRRCSeedJob: the only writer runs on push to main in mode remote +// with rrc not off, holds the only id-token permission in bazel.yml, reads +// no secret (its credential is the OIDC-minted certificate), sets Bazel up +// in local mode, seeds only with a certificate, uploads to the cache alone +// (no executor), and always removes the key. +func TestBazelRRCSeedJob(t *testing.T) { + wf := readCIWorkflow(t, bazelWorkflowName) + job, ok := wf.Jobs[bazelRRCSeedJob] + if !ok { + t.Fatalf("%s has no %s job", bazelWorkflowName, bazelRRCSeedJob) + } + if job.If != bazelRRCSeedJobIf { + t.Errorf("%s if %q, want %q", bazelRRCSeedJob, job.If, bazelRRCSeedJobIf) + } + if !slices.Equal(job.Needs, []string{bazelRBEJobName}) || job.Outputs != nil { + t.Errorf("%s needs %v outputs %v; it needs the rbe job alone and reports nothing to the gate", bazelRRCSeedJob, job.Needs, job.Outputs) + } + for _, c := range []struct { + event, ref, mode, rrc string + want bool + }{ + {"push", "refs/heads/main", "remote", "seed", true}, + {"push", "refs/heads/main", "remote", "canary", true}, + {"push", "refs/heads/main", "remote", "on", true}, + {"push", "refs/heads/main", "remote", "off", false}, + {"push", "refs/heads/main", "cache", "on", false}, + {"push", "refs/heads/other", "remote", "on", false}, + {"pull_request", "refs/pull/1/merge", "remote", "on", false}, + {"pull_request_target", "refs/heads/main", "remote", "on", false}, + {"merge_group", "refs/heads/gh-readonly-queue/main/pr-1-abc", "remote", "on", false}, + {"workflow_dispatch", "refs/heads/main", "remote", "on", false}, + {"schedule", "refs/heads/main", "remote", "on", false}, + } { + got := evalRRCIf(t, job.If, map[string]string{ + "github.event_name": c.event, "github.ref": c.ref, "needs.rbe.outputs.mode": c.mode, "needs.rbe.outputs.rrc": c.rrc, + }) + if got != c.want { + t.Errorf("%+v: seed runs %v, want %v", c, got, c.want) + } + } + if want := map[string]any{"contents": "read", "id-token": "write"}; !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("%s permissions %v, want %v", bazelRRCSeedJob, job.Permissions, want) + } + raw := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName) + if text := bazelRRCJobText(t, raw, bazelRRCSeedJob); strings.Contains(text, "secrets.") { + t.Errorf("%s reads a secret; its only credential is the OIDC-minted writer certificate", bazelRRCSeedJob) + } + setup := rrcStep(t, job, "setup-bazel", func(s ciWorkflowStep) bool { return s.Uses == bazelSetupBazelUses }) + if len(setup.Env) != 0 { + t.Errorf("%s setup-bazel env %v; local mode (no executor, no certificate) only", bazelRRCSeedJob, setup.Env) + } + cred := rrcStep(t, job, "writer certificate", func(s ciWorkflowStep) bool { return s.ID == "writer" }) + if cred.Run != "bash "+bazelRRCCredential { + t.Errorf("writer step runs %q, want bash %s", cred.Run, bazelRRCCredential) + } + seed := rrcStep(t, job, "seed", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, bazelRRCRunLanes) }) + if seed.If != "${{ steps.writer.outputs.cert != '' }}" { + t.Errorf("seed step if %q; it must skip when the mint switched seeding off (no certificate)", seed.If) + } + for _, want := range []string{ + "bash " + bazelRRCRunLanes + " --experimental_remote_repo_contents_cache --", "--loading_phase_threads=64", + `"--remote_cache=$ENDPOINT"`, `"--remote_instance_name=$INSTANCE"`, + `"--tls_client_certificate=$CERT"`, `"--tls_client_key=$KEY"`, "--remote_upload_local_results", + } { + if !strings.Contains(seed.Run, want) { + t.Errorf("seed step lacks %q:\n%s", want, seed.Run) + } + } + if strings.Contains(seed.Run, "remote_executor") { + t.Errorf("seed step names an executor; the writer certificate is cache only") + } + cleanup := rrcStep(t, job, "key removal", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, "rrc-writer.key") }) + if cleanup.If != "${{ always() }}" { + t.Errorf("the writer key removal runs if %q; it must always run", cleanup.If) + } +} + +// bazelRRCJobText returns a job's raw text in a workflow. +func bazelRRCJobText(t *testing.T, raw, job string) string { + t.Helper() + start := strings.Index(raw, "\n "+job+":\n") + if start < 0 { + t.Fatalf("cannot find the %s job's text", job) + } + text := raw[start+1:] + if next := regexp.MustCompile(`\n [a-z0-9-]+:\n`).FindStringIndex(text); next != nil { + text = text[:next[0]] + } + return text +} + +// TestBazelRRCVerifyJob: the nightly check runs from nightly.yml (schedule) +// or a dispatch while the cache is in use, fetches every lane cold without +// the repo contents cache, compares with rrc_verify.py using the lanes' +// read-only certificate and setup-bazel's endpoint (no secret outside +// setup-bazel), and alerts through an issue only when the comparison fails. +func TestBazelRRCVerifyJob(t *testing.T) { + wf := readCIWorkflow(t, bazelWorkflowName) + job, ok := wf.Jobs[bazelRRCVerifyJob] + if !ok { + t.Fatalf("%s has no %s job", bazelWorkflowName, bazelRRCVerifyJob) + } + if job.If != bazelRRCVerifyJobIf { + t.Errorf("%s if %q, want %q", bazelRRCVerifyJob, job.If, bazelRRCVerifyJobIf) + } + if job.Outputs != nil { + t.Errorf("%s outputs %v; it reports nothing to the gate", bazelRRCVerifyJob, job.Outputs) + } + for _, c := range []struct { + event, mode, rrc string + want bool + }{ + {"schedule", "remote", "on", true}, + {"schedule", "remote", "seed", true}, + {"workflow_dispatch", "remote", "canary", true}, + {"schedule", "remote", "off", false}, + {"schedule", "cache", "on", false}, + {"push", "remote", "on", false}, + {"pull_request", "remote", "on", false}, + {"pull_request_target", "remote", "on", false}, + {"merge_group", "remote", "on", false}, + } { + got := evalRRCIf(t, job.If, map[string]string{ + "github.event_name": c.event, "needs.rbe.outputs.mode": c.mode, "needs.rbe.outputs.rrc": c.rrc, + }) + if got != c.want { + t.Errorf("%+v: verify runs %v, want %v", c, got, c.want) + } + } + if want := map[string]any{"contents": "read", "issues": "write"}; !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("%s permissions %v, want %v", bazelRRCVerifyJob, job.Permissions, want) + } + for _, s := range job.Steps { + if strings.Contains(s.Run, "experimental_remote_repo_contents_cache") || strings.Contains(s.Run, ".bazelrc.local") || + strings.Contains(s.Run, "remote_upload_local_results") { + t.Errorf("%s step %q reads or writes the repo contents cache; the cold fetch must not", bazelRRCVerifyJob, s.Name) + } + for k, v := range s.Env { + if strings.Contains(v, "secrets.") && s.Uses != bazelSetupBazelUses { + t.Errorf("%s step %q env %s reads a secret; only setup-bazel may", bazelRRCVerifyJob, s.Name, k) + } + } + } + fetch := rrcStep(t, job, "cold fetch", func(s ciWorkflowStep) bool { return strings.HasPrefix(s.Name, "Cold fetch") }) + if fetch.Run != "bash "+bazelRRCRunLanes+" -- --loading_phase_threads=64" { + t.Errorf("cold fetch runs %q; it must run every lane's command with no startup flag", fetch.Run) + } + verify := rrcStep(t, job, "verify", func(s ciWorkflowStep) bool { return s.ID == "verify" }) + for _, want := range []string{"python3 " + bazelRRCVerifyTool, `--cert "$secret_dir/client.crt"`, `--key "$secret_dir/client.key"`, "--instance oss", ".bazelversion", "remote_executor="} { + if !strings.Contains(verify.Run, want) { + t.Errorf("verify step lacks %q", want) + } + } + alert := rrcStep(t, job, "alert", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, "gh issue") }) + if alert.If != "${{ failure() && steps.verify.outcome == 'failure' }}" { + t.Errorf("alert step if %q; it must alert only on a failed comparison", alert.If) + } + if strings.Contains(alert.Run, "${{") { + t.Errorf("alert step interpolates an expression into its script; pass it through env") + } +} + +// permissionRank orders GitHub token permission levels so a grant can be +// compared with a request: write covers read, read covers none. +var permissionRank = map[string]int{"": 0, "none": 0, "read": 1, "write": 2} + +type permWorkflow struct { + Jobs map[string]struct { + Uses string `yaml:"uses"` + Permissions map[string]string `yaml:"permissions"` + } `yaml:"jobs"` +} + +// TestBazelWorkflowCallersGrantEveryRequestedPermission guards against +// startup_failure (gascity #7478): GitHub validates a reusable workflow's +// job permissions when the caller starts, including jobs whose `if:` would +// skip, so every workflow that calls bazel.yml must grant at least the +// highest level any bazel.yml job requests for each scope. +func TestBazelWorkflowCallersGrantEveryRequestedPermission(t *testing.T) { + root := sourceRepoRoot(t) + load := func(name string) permWorkflow { + var wf permWorkflow + if err := yaml.Unmarshal([]byte(readPolicyFile(t, root, ".github/workflows/"+name)), &wf); err != nil { + t.Fatalf("parse %s: %v", name, err) + } + return wf + } + need := map[string]string{} + for _, job := range load(bazelWorkflowName).Jobs { + for scope, level := range job.Permissions { + if permissionRank[level] > permissionRank[need[scope]] { + need[scope] = level + } + } + } + if need["id-token"] != "write" || need["issues"] != "write" { + t.Fatalf("bazel.yml requests %v; the parser is broken or the rrc jobs are gone", need) + } + entries, err := os.ReadDir(filepath.Join(root, ".github", "workflows")) + if err != nil { + t.Fatal(err) + } + var callers []string + for _, e := range entries { + name := e.Name() + if name == bazelWorkflowName || (!strings.HasSuffix(name, ".yml") && !strings.HasSuffix(name, ".yaml")) { + continue + } + for jobName, job := range load(name).Jobs { + if job.Uses != "./.github/workflows/"+bazelWorkflowName { + continue + } + callers = append(callers, name) + var missing []string + for scope, level := range need { + if permissionRank[job.Permissions[scope]] < permissionRank[level] { + missing = append(missing, scope+": "+level) + } + } + sort.Strings(missing) + if len(missing) > 0 { + t.Errorf("%s job %q calls %s without granting %s (startup_failure)", name, jobName, bazelWorkflowName, strings.Join(missing, ", ")) + } + } + } + sort.Strings(callers) + if want := []string{"bazel-farm.yml", "nightly.yml", "pr.yml"}; !slices.Equal(callers, want) { + t.Errorf("bazel.yml callers %v, want %v", callers, want) + } +} + +// rrcMintCurlStub stands in for curl in rrc-writer-credential.sh. The OIDC +// request (its URL carries audience=) logs the audience and authorization +// and answers a token. A mint request logs its Authorization header and +// answers as the space-separated RBE_TEST_MINT says, one word per request +// (the last repeats): ok (a certificate for the runner's key, CN +// rbe-rrc-writer-beads O=gascity), cn, endpoint, instance, otherkey, nocert +// (each wrong in that one way), 000 (connection refused) or an HTTP status +// with an error body. +const rrcMintCurlStub = `#!/usr/bin/env bash +set -euo pipefail +out= url= headers=() +while [ $# -gt 0 ]; do + case "$1" in + -o) out=$2; shift 2 ;; + -H) headers+=("$2"); shift 2 ;; + -w | --data | --connect-timeout | --max-time | --retry) shift 2 ;; + -*) shift ;; + *) url=$1; shift ;; + esac +done +case "$url" in +*audience=*) + echo "oidc audience=${url##*audience=} auth=${headers[0]}" >>"$RBE_TEST_MINT_LOG" + echo '{"value": "oidc-jwt-value"}' + exit 0 + ;; +esac +echo "mint $url auth=${headers[0]}" >>"$RBE_TEST_MINT_LOG" +n=$(grep -c '^mint ' "$RBE_TEST_MINT_LOG") +read -r -a answers <<<"$RBE_TEST_MINT" +i=$((n - 1)); [ "$i" -lt "${#answers[@]}" ] || i=$((${#answers[@]} - 1)) +answer=${answers[$i]} +key="$BAZEL_CI_SECRET_DIR/rrc-writer.key" cn=rbe-rrc-writer-beads endpoint=grpcs://rbe-west.ops.gascity.com:443 instance=oss +case "$answer" in +ok | cn | endpoint | instance | otherkey | nocert) ;; +000) echo "curl: (7) Failed to connect" >&2; exit 7 ;; +*) printf '{"error": "stub %s"}' "$answer" >"$out"; printf '%s' "$answer"; exit 0 ;; +esac +case "$answer" in +cn) cn=rbe-ci ;; +endpoint) endpoint=grpcs://attacker.example:443 ;; +instance) instance=main ;; +otherkey) openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out other.key 2>/dev/null; key=other.key ;; +esac +pem= +if [ "$answer" != nocert ]; then + pem=$(openssl req -new -x509 -key "$key" -subj "/O=gascity/CN=$cn" -days 1 2>/dev/null) +fi +jq -cn --arg pem "$pem" --arg e "$endpoint" --arg i "$instance" --arg cn "$cn" \ + '{cert_pem: $pem, endpoint: $e, instance: $i, cn: $cn}' >"$out" +printf 200 +` + +// TestRRCWriterCredential runs rrc-writer-credential.sh against a stubbed +// curl (rrcMintCurlStub): it asks GitHub for an OIDC token with audience +// rbe-rrc-writer, masks it, sends it to the mint, retries only 429, 502 and +// the network, treats 503 or an unreachable mint as seeding off (no +// outputs, exit 0), and accepts only a certificate for the key it generated, +// CN rbe-rrc-writer[-x] O=gascity, for rbe-west's endpoint and instance oss. +func TestRRCWriterCredential(t *testing.T) { + for _, tool := range []string{"bash", "jq", "openssl"} { + requireHostTool(t, tool) + } + script := filepath.Join(sourceRepoRoot(t), bazelRRCCredential) + for _, c := range []struct { + answers string + ok, off bool + mints int + }{ + {"ok", true, false, 1}, + {"429 ok", true, false, 2}, + {"502 ok", true, false, 2}, + {"000 ok", true, false, 2}, + {"502", false, false, 4}, + {"403", false, false, 1}, + {"401", false, false, 1}, + {"409", false, false, 1}, + {"cn", false, false, 1}, + {"endpoint", false, false, 1}, + {"instance", false, false, 1}, + {"otherkey", false, false, 1}, + {"nocert", false, false, 1}, + {"503", true, true, 1}, + {"000", true, true, 4}, + } { + dir := t.TempDir() + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + for name, body := range map[string]string{"curl": rrcMintCurlStub, "sleep": "#!/bin/sh\nexit 0\n"} { + if err := os.WriteFile(filepath.Join(bin, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + secret := filepath.Join(t.TempDir(), "secret") + output := filepath.Join(dir, "output") + out, err := runRRCScript(t, dir, "bash "+script, map[string]string{ + "PATH": bin + string(os.PathListSeparator) + os.Getenv("PATH"), + "BAZEL_CI_SECRET_DIR": secret, + "GITHUB_OUTPUT": output, + "ACTIONS_ID_TOKEN_REQUEST_URL": "https://token.invalid/oidc?api-version=2.0", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token", + "RBE_TEST_MINT": c.answers, + "RBE_TEST_MINT_LOG": filepath.Join(dir, "mint.log"), + }) + if (err == nil) != c.ok { + t.Errorf("%q: ok=%v, want %v\n%s", c.answers, err == nil, c.ok, out) + continue + } + log := readPolicyFile(t, dir, "mint.log") + if !strings.Contains(log, "oidc audience=rbe-rrc-writer auth=Authorization: bearer request-token\n") { + t.Errorf("%q: OIDC request %q", c.answers, log) + } + if got := strings.Count(log, "mint https://rbe-mint.ops.gascity.com:8444/v1/rrc-writer/cert auth=Authorization: Bearer oidc-jwt-value\n"); got != c.mints { + t.Errorf("%q: %d mint requests with the OIDC token, want %d:\n%s", c.answers, got, c.mints, log) + } + if !strings.Contains(out, "::add-mask::oidc-jwt-value") { + t.Errorf("%q: the OIDC token is not masked:\n%s", c.answers, out) + } + if !c.ok { + continue + } + if c.off { + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Errorf("%q: wrote outputs (%v); the seed must skip", c.answers, err) + } + if !strings.Contains(out, "seeding off (HTTP "+c.answers[len(c.answers)-3:]+")") { + t.Errorf("%q: no seeding-off warning:\n%s", c.answers, out) + } + continue + } + got := rrcOutputs(t, output) + for name, want := range map[string]string{ + "cert": filepath.Join(secret, "rrc-writer.crt"), "key": filepath.Join(secret, "rrc-writer.key"), + "endpoint": "grpcs://rbe-west.ops.gascity.com:443", "instance": "oss", + } { + if got[name] != want { + t.Errorf("%q: output %s=%q, want %q", c.answers, name, got[name], want) + } + } + if key := readPolicyFile(t, secret, "rrc-writer.key"); !strings.Contains(key, "-----BEGIN PRIVATE KEY-----") { + t.Errorf("%q: key is not PKCS#8 (Bazel's TLS refuses SEC1)", c.answers) + } + if fi, err := os.Stat(filepath.Join(secret, "rrc-writer.key")); err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("%q: key mode %v (%v), want 0600", c.answers, fi, err) + } + if _, err := os.Stat(filepath.Join(secret, "rrc-mint.json")); !os.IsNotExist(err) { + t.Errorf("%q: the mint reply stays on disk (%v)", c.answers, err) + } + } + + dir := t.TempDir() + if out, err := runRRCScript(t, dir, "bash "+script, map[string]string{ + "BAZEL_CI_SECRET_DIR": filepath.Join(t.TempDir(), "secret"), "GITHUB_OUTPUT": filepath.Join(dir, "output"), + }); err == nil || !strings.Contains(out, "id-token: write") { + t.Errorf("without an OIDC request URL: err %v, want a failure naming id-token: write\n%s", err, out) + } +} + +// The writer credential script and rrc_verify.py with its unit test are byte +// copies of gascity's (gascity 78c0e5c6f198306b96fe854c18b4d72f16395ce7, +// #7467), canonical there like tools/bazel/ci_analytics_extract.py: the +// pins catch a local edit that would drift out of lockstep. +var bazelRRCVendored = map[string]string{ + bazelRRCCredential: "021788a724876bd1df7e5c117a4b3e1e60c5b25bb18bca472d61c35a08cb1fac", + bazelRRCVerifyTool: "00d50e569e2a50d8ef3d2740460de7337b71afd14dc76d2d2442dbd162236f94", + bazelRRCVerifyTest: "d934f13b466660f3148cf3c8dc7273803b4417f2a52f4edc2dd2f43afb93bfaa", +} + +func TestBazelRRCVendoredFromGascity(t *testing.T) { + root := sourceRepoRoot(t) + for path, want := range bazelRRCVendored { + sum := sha256.Sum256([]byte(readPolicyFile(t, root, path))) + if got := hex.EncodeToString(sum[:]); got != want { + t.Errorf("%s sha256 = %s, want %s: it drifted from gascity's copy, or gascity changed it and this pin (and the copy) needs updating", path, got, want) + } + } +} + +// TestRRCVerifyUnitTests runs gascity's rrc_verify_test.py in place: the +// AC key construction pinned against Bazel 9.3.0's own writes, the +// roll-forward of intermediate entries, a poisoned entry, and a GUID for +// .bazelversion's release. +func TestRRCVerifyUnitTests(t *testing.T) { + root := sourceRepoRoot(t) + python := requireHostTool(t, "python3") + cmd := exec.Command(python, filepath.Join(root, bazelRRCVerifyTest), "-v") + cmd.Dir = filepath.Join(root, "tools", "bazel") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%s failed: %v\n%s", bazelRRCVerifyTest, err, out) + } +} diff --git a/scripts/ci_analytics_workflow_test.go b/scripts/ci_analytics_workflow_test.go index e76b9ad02..1dfbb353a 100644 --- a/scripts/ci_analytics_workflow_test.go +++ b/scripts/ci_analytics_workflow_test.go @@ -34,6 +34,8 @@ var ciAnalyticsExcludedJobs = map[string]string{ "package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract", "package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract", "bazel-release-cross": "one bazel-release-cross-compile.sh run with no BEP/exec-log/profile capture, as when it was a bazel-pure step", + "rrc-seed": "the remote repo contents cache writer: --nobuild analysis on push to main, no test lane to summarize", + "rrc-verify": "the remote repo contents cache's nightly check: --nobuild cold fetches, no test lane to summarize", } // ciAnalyticsRunTemplateRE finds a GitHub Actions expression (${{ ... }}) diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index d9b6e5fbc..77236fb98 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -580,8 +580,8 @@ func TestBlacksmithBazelRunnerSizes(t *testing.T) { for _, mode := range []string{"remote", "fork-ro", "fork-rw", "local", "cache", "skip"} { ctx := map[string]string{"needs.rbe.outputs.mode": mode} for name, job := range workflow.Jobs { - if name == bazelRBEJobName { - continue + if name == bazelRBEJobName || isBazelRRCJob(name) { + continue // rrc jobs: push/schedule only, never a fork (literal label) } want := "ubuntu-latest" if mode == "remote" { diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 5775326b9..24b8d8272 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1779,7 +1779,7 @@ const ( // bazel.yml's jobs: the rbe job that decides the execution mode, the // --config=ci lane, one job per CI job a Bazel config mirrors, and the two // package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings. -var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName} +var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName, bazelRRCSeedJob, bazelRRCVerifyJob} // The lanes that only run remotely (skipped unless the rbe job chose remote); // bazel-integration runs remotely or with the read-only cache (bazelIntegIf); @@ -2148,6 +2148,9 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) { t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName) } + if isBazelRRCJob(name) { + continue // runner, if and setup-bazel: bazel_rrc_test.go + } // F3: the package gates use larger runners (bazelPackageRunsOn) and // their own if (the caller's package-gates input, not the rbe job's // mode - they never skip for execution-mode reasons). @@ -2711,8 +2714,8 @@ func TestBazelLaneIsGated(t *testing.T) { "rbe-mode": "${{ jobs." + bazelRBEJobName + ".outputs.mode }}", } for name, job := range workflow.Jobs { - if name == bazelRBEJobName { - continue + if name == bazelRBEJobName || isBazelRRCJob(name) { + continue // the rrc jobs never run in a PR call (bazel_rrc_test.go) } _, gated := bazelLaneGateIDs[name] _, advisory := bazelAdvisoryLanes[name] @@ -2756,14 +2759,15 @@ func TestBazelLaneIsGated(t *testing.T) { } // One caller on PR events: pr.yml's bazel job, with the four RBE secrets - // only, read-only contents, and no if (the rbe job decides). + // only, bazelCallPermissions (contents: read, plus what the rrc jobs, + // which never run on a PR, ask for), and no if (the rbe job decides). pr := readCIWorkflow(t, "pr.yml") bazel := pr.job(t, "bazel") if bazel.Uses != "./.github/workflows/"+bazelWorkflowName || bazel.If != "" || len(bazel.Needs) != 0 { t.Errorf("pr.yml bazel job uses=%q if=%q needs=%v; want an unconditional call of %s", bazel.Uses, bazel.If, bazel.Needs, bazelWorkflowName) } - if !reflect.DeepEqual(bazel.Permissions, map[string]any{"contents": "read"}) { - t.Errorf("pr.yml bazel job permissions = %v, want contents: read", bazel.Permissions) + if !reflect.DeepEqual(bazel.Permissions, bazelCallPermissions) { + t.Errorf("pr.yml bazel job permissions = %v, want %v", bazel.Permissions, bazelCallPermissions) } if !reflect.DeepEqual(bazel.With, bazelPRCallWith) { t.Errorf("pr.yml bazel job with = %v, want exactly %v (no rbe or other override)", bazel.With, bazelPRCallWith) @@ -3039,6 +3043,11 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string // bazelAdvisoryLanes' name-specific exception in // TestBazelGateSimulation below). return map[string]bool{"remote": true} + case bazelRRCSeedJobIf, bazelRRCVerifyJobIf: + // The remote repo contents cache's writer (push to main) and + // nightly check (schedule, dispatch): never in a pull_request, + // merge_group or pull_request_target call, in any mode. + return map[string]bool{} } t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr) return nil @@ -4182,12 +4191,13 @@ func TestBazelDoltServerTiers(t *testing.T) { }{ // +2 over the earlier counts: the CI analytics summary/upload steps // (S3; see bazel-test's "CI analytics summary" comment), added to - // every lane job ahead of the result recorder. - {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 10}, + // every lane job ahead of the result recorder. +1: the remote repo + // contents cache reader (TestBazelRRCReadSteps). + {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 11}, // One more than bazel-proxied: a "Shard balance" step (rbe-ci-cost- // latency-study.md recommendation 4), since this is the tier whose // last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. - {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 11}, + {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 12}, } { job := workflow.job(t, c.job) if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 { @@ -4195,7 +4205,7 @@ func TestBazelDoltServerTiers(t *testing.T) { } assertBazelTierStep(t, job, c.job, c.config) if n := len(job.Steps); n != c.wantSteps { - t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps) + t.Errorf("%s has %d steps, want %d (checkout, the rrc reader, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps) } logs := job.step(t, "Upload test logs") if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { @@ -4787,12 +4797,15 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { "enabled": "${{ steps.decide.outputs.enabled }}", "mode": "${{ steps.decide.outputs.mode }}", "tier": "${{ steps.decide.outputs.tier }}", + // The remote repo contents cache's switch, from decide's mode + // (TestBazelRRCModeStep). + "rrc": "${{ steps.rrc.outputs.rrc }}", } if !reflect.DeepEqual(job.Outputs, wantOutputs) { t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs) } - if len(job.Steps) != 3 { - t.Fatalf("%s has %d steps, want the decision step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps)) + if len(job.Steps) != 4 { + t.Fatalf("%s has %d steps, want the decision step, the rrc step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps)) } step := job.Steps[0] wantEnv := map[string]string{ diff --git a/scripts/rbe_worker_env_test.go b/scripts/rbe_worker_env_test.go index a9d849f24..8958859f2 100644 --- a/scripts/rbe_worker_env_test.go +++ b/scripts/rbe_worker_env_test.go @@ -367,21 +367,21 @@ func TestRBEWorkerEnvSync(t *testing.T) { // mode the lanes read is decide's alone. func TestBazelRBEWorkerEnvPreflight(t *testing.T) { job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName) - if len(job.Steps) != 3 || job.Steps[0].ID != "decide" { - t.Fatalf("%s steps = %d (first %q), want decide, the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID) + if len(job.Steps) != 4 || job.Steps[0].ID != "decide" || job.Steps[1].ID != bazelRRCModeStepID { + t.Fatalf("%s steps = %d (first %q), want decide, rrc (TestBazelRRCModeStep), the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID) } if !strings.Contains(job.Steps[0].Run, `case "$mode" in remote|fork-ro|fork-rw) enabled=true ;; esac`) { t.Errorf("%s decide no longer sets enabled exactly for the remote modes; the preflight's if depends on it", bazelRBEJobName) } - checkout, check := job.Steps[1], job.Steps[2] + checkout, check := job.Steps[2], job.Steps[3] if actionFamily(checkout.Uses) != "actions/checkout" || checkout.If != rbeWorkerEnvEnabledIf || checkout.With["sparse-checkout"] != bazelRBESparseCheckout { - t.Errorf("%s step 2: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s", + t.Errorf("%s step 3: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s", bazelRBEJobName, checkout.Uses, checkout.If, checkout.With["sparse-checkout"], bazelRBESparseCheckout, rbeWorkerEnvEnabledIf) } wantEnv := map[string]string{"MODE": "${{ steps.decide.outputs.mode }}"} if check.If != rbeWorkerEnvEnabledIf || check.ID != "" || check.Uses != "" || check.ContinueOnError != nil || !reflect.DeepEqual(check.Env, wantEnv) || !strings.Contains(check.Run, "tools/rbe/worker-env-sync main") { - t.Errorf("%s step 3: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main", + t.Errorf("%s step 4: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main", bazelRBEJobName, check.If, check.ID, check.Uses, check.ContinueOnError, check.Env, rbeWorkerEnvEnabledIf, wantEnv) } @@ -427,8 +427,11 @@ func TestNightlyWorkerEnvSyncOpensIssue(t *testing.T) { } for name, job := range nightly.Jobs { want := any(nil) - if name == nightlyWorkerEnvJob { + switch name { + case nightlyWorkerEnvJob: want = map[string]any{"contents": "read", "issues": "write"} + case "bazel": // the bazel.yml call: what its rrc jobs ask for + want = bazelCallPermissions } if !reflect.DeepEqual(job.Permissions, want) { t.Errorf("nightly.yml job %s permissions = %v, want %v", name, job.Permissions, want) diff --git a/scripts/tracked_files_test.go b/scripts/tracked_files_test.go index 1fd09c3af..fddeeea12 100644 --- a/scripts/tracked_files_test.go +++ b/scripts/tracked_files_test.go @@ -67,6 +67,16 @@ const ( publicRBEForkPinLine = `ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-fork\.ops\.gascity\.com:8444$'}` ) +// publicRRCWriterPin: rrc-writer-credential.sh (a byte copy of gascity's) +// pins the one endpoint rbe-west's mint may hand the remote repo contents +// cache writer (rbe-west, :443), so a compromised mint cannot point the +// writer's uploads elsewhere. Public, no credential: allowed as exactly +// this line, in exactly that file. +const ( + publicRRCWriterPinFile = ".github/scripts/rrc-writer-credential.sh" + publicRRCWriterPinLine = `ENDPOINT_RE=${RBE_RRC_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-west\.ops\.gascity\.com(:443)?$'}` +) + type endpointHit struct { path string line int @@ -121,7 +131,8 @@ func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] { continue } - if path == publicRBEForkPinFile && line == publicRBEForkPinLine { + if path == publicRBEForkPinFile && line == publicRBEForkPinLine || + path == publicRRCWriterPinFile && line == publicRRCWriterPinLine { continue } for _, re := range flagRes { diff --git a/tools/bazel/rrc_verify.py b/tools/bazel/rrc_verify.py new file mode 100644 index 000000000..779b49483 --- /dev/null +++ b/tools/bazel/rrc_verify.py @@ -0,0 +1,620 @@ +#!/usr/bin/env python3 +"""Verify rbe-west's remote repo contents cache against a cold fetch (ga-vnycm2.26). + +Bazel's --experimental_remote_repo_contents_cache stores each reproducible +external repository as the ActionResult of a synthetic, never executed action +keyed by the repository rule's predeclared-inputs hash +(RemoteRepoContentsCacheImpl, engdocs/design/bazel-remote-repo-contents-cache.md). +Nothing on the reader checks that the cached tree is what the rule produces: +that is the work the cache skips. So, nightly, this job fetches every lane's +repositories cold, without the cache, and compares each repository's tree and +marker file with the entry a lane would read for the same key. A mismatch is +a poisoned entry or a repository rule falsely marked reproducible (design +R1/R2); both must be looked at by a human. + +Inputs: the cold output base (setup-bazel's rc: --repo_contents_cache= keeps +each repository in external/ with its marker external/@.marker) +and read access to the cache (rbe-west's trusted edge with the CI client +certificate, or any REAPI endpoint). For each marker, it walks the action +cache as the reader does: the final entry under the predeclared hash, or +intermediate entries whose stdout lists batches of recorded inputs, whose +values it takes from the cold marker and rolls into the next key +(Fingerprint: sha256 over length-prefixed UTF-8 strings). The final entry's +marker must equal the cold marker byte for byte, and its tree (the Tree +proto, read from the CAS) must hold the same names, file digests, executable +bits and symlink targets as the cold directory. + +Transport: unary and server-streaming gRPC over curl's HTTP/2 with the +standard library's protobuf-free wire decoding, as tools/rbe/cache-zstd-probe.sh +does: no grpc package on a job that holds a farm credential. + +Exit status: 0 every cached repository matches; 1 a mismatch or an invalid +entry; 2 a usage error or too many transport failures to tell. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import dataclasses +import hashlib +import json +import os +import stat +import subprocess +import sys +import tempfile +from typing import Callable, Iterable, Optional + +# RemoteRepoContentsCacheImpl.GUID per Bazel release: the synthetic Command's +# only argument, so every key depends on it. A .bazelversion bump adds its +# release here (scripts/bazel_rrc_test.go checks .bazelversion has one). +BAZEL_RRC_GUIDS = { + "9.3.0": "06a53d89-9f52-46ed-8064-f7af6ba27769", +} +MARKER_FILE_PATH = ".recorded_inputs" +REPO_DIRECTORY_PATH = "repo_contents" +EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() +GRPC_NOT_FOUND = 5 +MAX_CHAIN = 64 # intermediate hops per repository +MAX_DIFFS = 20 # per repository, in the report + + +# --- protobuf wire format --------------------------------------------------- + + +def _varint(n: int) -> bytes: + out = bytearray() + while True: + b = n & 0x7F + n >>= 7 + if n: + out.append(b | 0x80) + else: + out.append(b) + return bytes(out) + + +def _field(num: int, payload: bytes) -> bytes: + return _varint(num << 3 | 2) + _varint(len(payload)) + payload + + +def _field_varint(num: int, v: int) -> bytes: + return _varint(num << 3) + _varint(v) + + +def _read_varint(b: bytes, i: int) -> tuple[int, int]: + v = s = 0 + while True: + if i >= len(b) or s > 63: + raise ValueError("truncated varint") + c = b[i] + i += 1 + v |= (c & 0x7F) << s + s += 7 + if c < 0x80: + return v, i + + +def fields(b: bytes) -> Iterable[tuple[int, int, object]]: + """Yields (field number, wire type, value) for one message.""" + i = 0 + while i < len(b): + key, i = _read_varint(b, i) + num, wire = key >> 3, key & 7 + if wire == 0: + v, i = _read_varint(b, i) + elif wire == 1: + v, i = b[i : i + 8], i + 8 + elif wire == 2: + n, i = _read_varint(b, i) + v, i = b[i : i + n], i + n + elif wire == 5: + v, i = b[i : i + 4], i + 4 + else: + raise ValueError("wire type %d" % wire) + if i > len(b): + raise ValueError("truncated field %d" % num) + yield num, wire, v + + +@dataclasses.dataclass(frozen=True) +class Digest: + hash: str + size: int + + def encode(self) -> bytes: + out = _field(1, self.hash.encode()) + if self.size: + out += _field_varint(2, self.size) + return out + + @staticmethod + def decode(b: bytes) -> "Digest": + h, size = "", 0 + for num, _, v in fields(b): + if num == 1: + h = bytes(v).decode() + elif num == 2: + size = int(v) + return Digest(h, size) + + @staticmethod + def of(data: bytes) -> "Digest": + return Digest(hashlib.sha256(data).hexdigest(), len(data)) + + +def command_bytes(guid: str) -> bytes: + """RemoteRepoContentsCacheImpl.COMMAND, serialized in field order.""" + return ( + _field(1, guid.encode()) # arguments + + _field(3, MARKER_FILE_PATH.encode()) # output_files + + _field(4, REPO_DIRECTORY_PATH.encode()) # output_directories + + _field(5, b"") # platform: the default instance, set + + _field(7, MARKER_FILE_PATH.encode()) # output_paths + + _field(7, REPO_DIRECTORY_PATH.encode()) + ) + + +def action_digest(guid: str, input_hash: str) -> Digest: + """The AC key the cache uses for input_hash (buildAction).""" + action = ( + _field(1, Digest.of(command_bytes(guid)).encode()) # command_digest + + _field(2, Digest(EMPTY_SHA256, 0).encode()) # input_root_digest + + _field(9, input_hash.encode("latin-1")) # salt + + _field(10, b"") # platform + ) + return Digest.of(action) + + +def _bazel_string(s: str) -> bytes: + # Bazel holds file contents read as ISO-8859-1 in Java Strings and + # Fingerprint.addString writes those Strings as UTF-8. + return s.encode("utf-8") + + +def roll_forward(hash_: str, recorded_input_with_value: str) -> str: + """RemoteRepoContentsCacheImpl.rollForwardHash.""" + out = hashlib.sha256() + for s in (hash_, recorded_input_with_value): + b = _bazel_string(s) + out.update(_varint(len(b)) + b) + return out.hexdigest() + + +# --- REAPI messages --------------------------------------------------------- + + +@dataclasses.dataclass +class ActionResult: + exit_code: int = 0 + output_files: list = dataclasses.field(default_factory=list) # (path, Digest, is_executable, contents) + output_directories: list = dataclasses.field(default_factory=list) # (path, tree Digest) + symlinks: int = 0 + stdout_raw: bytes = b"" + stdout_digest: Optional[Digest] = None + stderr_size: int = 0 + + @staticmethod + def decode(b: bytes) -> "ActionResult": + r = ActionResult() + for num, _, v in fields(b): + if num == 2: + path, dg, exe, contents = "", None, False, b"" + for n2, _, v2 in fields(v): + if n2 == 1: + path = bytes(v2).decode() + elif n2 == 2: + dg = Digest.decode(v2) + elif n2 == 4: + exe = bool(v2) + elif n2 == 5: + contents = bytes(v2) + r.output_files.append((path, dg, exe, contents)) + elif num == 3: + path, tree = "", None + for n2, _, v2 in fields(v): + if n2 == 1: + path = bytes(v2).decode() + elif n2 == 3: + tree = Digest.decode(v2) + r.output_directories.append((path, tree)) + elif num == 4: + r.exit_code = int(v) + elif num == 5: + r.stdout_raw = bytes(v) + elif num == 6: + r.stdout_digest = Digest.decode(v) + elif num in (7, 8): + r.stderr_size += len(v) + elif num in (10, 11, 12): + r.symlinks += 1 + return r + + +@dataclasses.dataclass +class Directory: + files: dict # name -> (Digest, is_executable) + dirs: dict # name -> Digest + symlinks: dict # name -> target + + @staticmethod + def decode(b: bytes) -> "Directory": + d = Directory({}, {}, {}) + for num, _, v in fields(b): + name, dg, exe, target = "", None, False, "" + for n2, _, v2 in fields(v): + if n2 == 1: + name = bytes(v2).decode("utf-8", "surrogateescape") + elif n2 == 2 and num in (1, 2): + dg = Digest.decode(v2) + elif n2 == 2 and num == 3: + target = bytes(v2).decode("utf-8", "surrogateescape") + elif n2 == 4 and num == 1: + exe = bool(v2) + if num == 1: + d.files[name] = (dg, exe) + elif num == 2: + d.dirs[name] = dg + elif num == 3: + d.symlinks[name] = target + return d + + +def decode_tree(b: bytes) -> tuple[Directory, dict]: + """Tree -> (root, {Directory digest hash: Directory}).""" + root, children = None, {} + for num, _, v in fields(b): + if num == 1: + root = Directory.decode(v) + elif num == 2: + children[hashlib.sha256(v).hexdigest()] = Directory.decode(v) + if root is None: + raise ValueError("Tree without a root") + return root, children + + +# --- transport ---------------------------------------------------------------- + + +class RPCError(Exception): + def __init__(self, code: int, message: str): + super().__init__("gRPC status %d: %s" % (code, message)) + self.code = code + + +def _frame(msg: bytes) -> bytes: + return b"\0" + len(msg).to_bytes(4, "big") + msg + + +def _unframe(body: bytes) -> list[bytes]: + out, i = [], 0 + while i < len(body): + if i + 5 > len(body) or body[i] != 0: + raise ValueError("not an uncompressed gRPC frame") + n = int.from_bytes(body[i + 1 : i + 5], "big") + out.append(body[i + 5 : i + 5 + n]) + i += 5 + n + if i != len(body): + raise ValueError("truncated gRPC frame") + return out + + +class CurlTransport: + """gRPC over curl's HTTP/2: grpcs:// with optional mTLS, or grpc://.""" + + def __init__(self, endpoint: str, cert: str = "", key: str = "", ca: str = "", timeout: int = 60): + if endpoint.startswith("grpcs://"): + self.base = "https://" + endpoint[len("grpcs://") :] + self.tls = True + elif endpoint.startswith("grpc://"): + self.base = "http://" + endpoint[len("grpc://") :] + self.tls = False + else: + raise ValueError("endpoint must be grpcs:// or grpc://, got %r" % endpoint) + self.base = self.base.rstrip("/") + self.cert, self.key, self.ca, self.timeout = cert, key, ca, timeout + + def call(self, method: str, request: bytes) -> list[bytes]: + with tempfile.TemporaryDirectory() as tmp: + req, head, body = (os.path.join(tmp, n) for n in ("req", "head", "body")) + with open(req, "wb") as f: + f.write(_frame(request)) + cmd = ["curl", "-sS", "--connect-timeout", "10", "--max-time", str(self.timeout)] + cmd += ["--http2"] if self.tls else ["--http2-prior-knowledge"] + if self.cert: + cmd += ["--cert", self.cert, "--key", self.key] + if self.ca: + cmd += ["--cacert", self.ca] + cmd += [ + "-H", "content-type: application/grpc", "-H", "te: trailers", + "--data-binary", "@" + req, "-D", head, "-o", body, "-w", "%{http_code}", + self.base + "/" + method, + ] # fmt: skip + p = subprocess.run(cmd, capture_output=True, text=True, check=False) + if p.returncode != 0: + raise RPCError(-1, "curl exit %d: %s" % (p.returncode, p.stderr.strip())) + if p.stdout != "200": + raise RPCError(-1, "HTTP %s" % p.stdout) + headers = read_bytes(head).decode("latin-1").replace("\r", "") + status, message = None, "" + for line in headers.split("\n"): + k, _, v = line.partition(":") + if k.strip().lower() == "grpc-status": + status = int(v.strip()) + elif k.strip().lower() == "grpc-message": + message = v.strip() + if status is None: + raise RPCError(-1, "no grpc-status") + if status != 0: + raise RPCError(status, message) + return _unframe(read_bytes(body)) + + +class Cache: + """The few REAPI reads the reader makes.""" + + def __init__(self, rpc: Callable[[str, bytes], list[bytes]], instance: str): + self.rpc, self.instance = rpc, instance + + def action_result(self, key: Digest) -> Optional[ActionResult]: + req = ( + _field(1, self.instance.encode()) + + _field(2, key.encode()) + + _field_varint(3, 1) # inline_stdout + + _field(5, MARKER_FILE_PATH.encode()) # inline_output_files + ) + try: + msgs = self.rpc("build.bazel.remote.execution.v2.ActionCache/GetActionResult", req) + except RPCError as e: + if e.code == GRPC_NOT_FOUND: + return None + raise + if len(msgs) != 1: + raise RPCError(-1, "GetActionResult: %d messages" % len(msgs)) + return ActionResult.decode(msgs[0]) + + def blob(self, d: Digest) -> bytes: + if d.size == 0: + return b"" + name = "%s/blobs/%s/%d" % (self.instance, d.hash, d.size) if self.instance else "blobs/%s/%d" % (d.hash, d.size) + msgs = self.rpc("google.bytestream.ByteStream/Read", _field(1, name.encode())) + data = b"".join(bytes(v) for m in msgs for num, _, v in fields(m) if num == 10) + if Digest.of(data) != d: + raise RPCError(-1, "blob %s/%d: got %d bytes that do not match" % (d.hash, d.size, len(data))) + return data + + +# --- verification --------------------------------------------------------------- + + +@dataclasses.dataclass +class Result: + repo: str + status: str # match, mismatch, invalid, not_cached, error + detail: list = dataclasses.field(default_factory=list) + + +def read_marker(path: str) -> tuple[str, dict]: + """(predeclared hash, {recorded input: its 'input value' line}).""" + text = read_bytes(path).decode("latin-1") + lines = text.split("\n") + recorded = {} + for line in lines[1:]: + if not line: + continue + inp, sep, _ = line.partition(" ") + if sep: + recorded[inp] = line + return lines[0].strip(), recorded + + +def find_final(cache: Cache, guid: str, predeclared: str, recorded: dict) -> tuple[Optional[ActionResult], list]: + """Walks the AC from predeclared like fetchFinalCacheEntry; returns (final entry or None, invalid reasons).""" + invalid, current, seen = [], [predeclared], set() + for _ in range(MAX_CHAIN): + nxt = [] + for h in current: + if h in seen: + continue + seen.add(h) + ar = cache.action_result(action_digest(guid, h)) + if ar is None: + continue + if ar.exit_code != 0: + invalid.append("entry %s: exit code %d" % (h, ar.exit_code)) + continue + if len(ar.output_files) == 1 and len(ar.output_directories) == 1 and ar.symlinks == 0: + return ar, invalid + if ar.output_files or ar.output_directories or ar.symlinks or not (ar.stdout_raw or (ar.stdout_digest and ar.stdout_digest.size)): + invalid.append("entry %s: neither a final nor an intermediate result" % h) + continue + stdout = ar.stdout_raw if ar.stdout_raw else cache.blob(ar.stdout_digest) + for batch in stdout.decode("latin-1").splitlines(): + rolling = h + for inp in batch.split(" "): + line = recorded.get(inp) + if line is None: + break # the cold fetch recorded other inputs: not this branch + rolling = roll_forward(rolling, line) + else: + nxt.append(rolling) + if not nxt: + return None, invalid + current = nxt + invalid.append("more than %d intermediate hops" % MAX_CHAIN) + return None, invalid + + +def read_bytes(path: str) -> bytes: + with open(path, "rb") as f: + return f.read() + + +def _sha256_file(path: str) -> tuple[str, int]: + h, n = hashlib.sha256(), 0 + with open(path, "rb") as f: + while chunk := f.read(1 << 20): + h.update(chunk) + n += len(chunk) + return h.hexdigest(), n + + +def compare_tree(root: Directory, children: dict, local: str) -> list: + """Differences between the cached tree and the cold directory local.""" + diffs = [] + + def walk(d: Directory, path: str, rel: str) -> None: + if len(diffs) >= MAX_DIFFS: + return + try: + names = set(os.listdir(path)) + except OSError as e: + diffs.append("%s: cannot list the cold directory (%s)" % (rel or ".", e)) + return + remote = set(d.files) | set(d.dirs) | set(d.symlinks) + for n in sorted(remote - names): + diffs.append("%s: only in the cache" % os.path.join(rel, n)) + for n in sorted(names - remote): + diffs.append("%s: only in the cold fetch" % os.path.join(rel, n)) + for n in sorted(remote & names): + if len(diffs) >= MAX_DIFFS: + return + p, r = os.path.join(path, n), os.path.join(rel, n) + st = os.lstat(p) + if n in d.symlinks: + if not stat.S_ISLNK(st.st_mode): + diffs.append("%s: a symlink in the cache, not in the cold fetch" % r) + elif os.readlink(p) != d.symlinks[n]: + diffs.append("%s: symlink to %r in the cache, %r cold" % (r, d.symlinks[n], os.readlink(p))) + continue + try: + st = os.stat(p) # the upload follows symlinks it does not keep + except OSError as e: + diffs.append("%s: dangling in the cold fetch (%s)" % (r, e)) + continue + if n in d.files: + dg, exe = d.files[n] + if not stat.S_ISREG(st.st_mode): + diffs.append("%s: a file in the cache, not in the cold fetch" % r) + continue + h, size = _sha256_file(p) + if dg is None or (h, size) != (dg.hash, dg.size): + diffs.append("%s: content differs (cache %s/%s, cold %s/%d)" % (r, dg and dg.hash, dg and dg.size, h, size)) + if exe != bool(st.st_mode & stat.S_IXUSR): + diffs.append("%s: executable bit differs (cache %s)" % (r, exe)) + else: + if not stat.S_ISDIR(st.st_mode): + diffs.append("%s: a directory in the cache, not in the cold fetch" % r) + continue + child = children.get(d.dirs[n].hash) + if child is None: + diffs.append("%s: the Tree lacks directory %s" % (r, d.dirs[n].hash)) + continue + walk(child, p, r) + + walk(root, local, "") + return diffs + + +def verify_repo(cache: Cache, guid: str, external: str, name: str) -> Result: + marker = os.path.join(external, "@%s.marker" % name) + try: + predeclared, recorded = read_marker(marker) + final, invalid = find_final(cache, guid, predeclared, recorded) + if invalid: + return Result(name, "invalid", invalid) + if final is None: + return Result(name, "not_cached") + diffs = [] + path, mdg, _, contents = final.output_files[0] + dpath, tree = final.output_directories[0] + if path != MARKER_FILE_PATH or dpath != REPO_DIRECTORY_PATH or tree is None or mdg is None: + return Result(name, "invalid", ["final entry outputs %r and %r" % (path, dpath)]) + cached_marker = contents if contents else cache.blob(mdg) + if cached_marker != read_bytes(marker): + diffs.append("%s: marker file differs" % MARKER_FILE_PATH) + root, children = decode_tree(cache.blob(tree)) + diffs += compare_tree(root, children, os.path.join(external, name)) + return Result(name, "mismatch" if diffs else "match", diffs[:MAX_DIFFS]) + except (RPCError, ValueError, OSError) as e: + return Result(name, "error", [str(e)]) + + +def repos(external: str) -> list: + out = [] + for f in sorted(os.listdir(external)): + if f.startswith("@") and f.endswith(".marker") and os.path.isdir(os.path.join(external, f[1:-7])): + out.append(f[1:-7]) + return out + + +def verify(cache: Cache, guid: str, output_base: str, jobs: int) -> list: + external = os.path.join(output_base, "external") + names = repos(external) + with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool: + return list(pool.map(lambda n: verify_repo(cache, guid, external, n), names)) + + +def summarize(results: list) -> dict: + counts = {} + for r in results: + counts[r.status] = counts.get(r.status, 0) + 1 + return counts + + +def exit_status(results: list, max_error_fraction: float) -> int: + counts = summarize(results) + if counts.get("mismatch") or counts.get("invalid"): + return 1 + if not results or counts.get("error", 0) > max_error_fraction * len(results): + return 2 + return 0 + + +def main(argv: Optional[list] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--output-base", required=True, help="the cold fetch's output base") + ap.add_argument("--endpoint", required=True, help="grpcs://host:port or grpc://host:port") + ap.add_argument("--instance", default="oss") + ap.add_argument("--cert", default="") + ap.add_argument("--key", default="") + ap.add_argument("--ca", default="") + ap.add_argument("--bazel-version", required=True, help="the release that wrote the entries (.bazelversion)") + ap.add_argument("--jobs", type=int, default=16) + ap.add_argument("--max-error-fraction", type=float, default=0.01, help="transport errors tolerated before exit 2") + ap.add_argument("--json", default="", help="write the full report here") + ap.add_argument("--summary", default="", help="append a Markdown summary here (GITHUB_STEP_SUMMARY)") + a = ap.parse_args(argv) + guid = BAZEL_RRC_GUIDS.get(a.bazel_version) + if guid is None: + print("rrc-verify: no GUID for Bazel %s; add RemoteRepoContentsCacheImpl.GUID to BAZEL_RRC_GUIDS" % a.bazel_version, file=sys.stderr) + return 2 + if bool(a.cert) != bool(a.key): + print("rrc-verify: --cert and --key go together", file=sys.stderr) + return 2 + cache = Cache(CurlTransport(a.endpoint, a.cert, a.key, a.ca).call, a.instance) + results = verify(cache, guid, a.output_base, a.jobs) + counts = summarize(results) + bad = [r for r in results if r.status in ("mismatch", "invalid", "error")] + lines = ["rrc-verify: %d repositories: %s" % (len(results), ", ".join("%s %d" % kv for kv in sorted(counts.items())))] + for r in bad: + lines.append(" %s %s: %s" % (r.status, r.repo, "; ".join(r.detail))) + print("\n".join(lines)) + if a.json: + with open(a.json, "w") as f: + json.dump({"counts": counts, "results": [dataclasses.asdict(r) for r in results if r.status != "match"]}, f, indent=1) + if a.summary: + with open(a.summary, "a") as f: + f.write("### Remote repo contents cache verification\n\n") + f.write("| status | repositories |\n|---|---|\n") + for k, v in sorted(counts.items()): + f.write("| %s | %d |\n" % (k, v)) + for r in bad[:50]: + f.write("\n- **%s** `%s`: %s" % (r.status, r.repo, "; ".join(r.detail)[:500])) + f.write("\n") + return exit_status(results, a.max_error_fraction) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/rrc_verify_test.py b/tools/bazel/rrc_verify_test.py new file mode 100644 index 000000000..ac5ef9eef --- /dev/null +++ b/tools/bazel/rrc_verify_test.py @@ -0,0 +1,323 @@ +"""Tests for tools/bazel/rrc_verify.py (ga-vnycm2.26). + +The golden keys below come from Bazel 9.3.0 itself: it wrote the entries +(`bazel test --nobuild --experimental_remote_repo_contents_cache +--remote_upload_local_results` into a throwaway NativeLink 1.7.1), and +rrc_verify found 1124 of 1124 cached repositories under the keys it computes. +Everything else runs against an in-memory cache. +""" + +import hashlib +import os +import stat +import tempfile +import unittest + +import rrc_verify as v + +GUID = v.BAZEL_RRC_GUIDS["9.3.0"] + +# gazelle++go_deps+com_github_aleksi_pointer, a go_repository: its marker +# (predeclared hash, then recorded inputs) and the chain of intermediate +# entries Bazel 9.3.0 wrote for it, one batch per hop. +POINTER_MARKER = ( + "d2e29605e9f6822328a5f4d0fcacfc80203bd61260f860e6f9a942ec96a5927b\n" + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//go.env f46153b50029d3d4a489ba97ee1b14e811f12f76a6e41fa6471c92ea6dc957a9\n" + "REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_cache gazelle++non_module_deps+bazel_gazelle_go_repository_cache\n" + "FILE:@@gazelle++non_module_deps+bazel_gazelle_go_repository_cache//go.env f46153b50029d3d4a489ba97ee1b14e811f12f76a6e41fa6471c92ea6dc957a9\n" + "REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_tools +go_repository_tools+gazelle_go_repository_tools\n" + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/fetch_repo 8b1288089311dc63c3d4b04c433d9ba11df5b09b5e8fdb1f8db0804f360ca6c0\n" + "ENV:GO_REPOSITORY_EPHEMERAL_MODCACHE \\0\n" + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/gazelle 1b764f6d9ad08e324bdcab429c3105815b8914f289d594acfc791bdac2bc2b2c\n" + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//WORKSPACE cc988347dd32755bac20223ea21cd982da6da90d9113dee8ccd4c4b88b3983db\n" +) +# (input hash, its AC key, the intermediate entry's stdout) +POINTER_CHAIN = [ + ("d2e29605e9f6822328a5f4d0fcacfc80203bd61260f860e6f9a942ec96a5927b", + "add188cbb3ec90ac802738e80e42516a17ba75e7563efffa80f64fee38edf866", + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//go.env REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_cache\n"), + ("8bc57512298f35dd223a75991454f01b20e0581fc5b3cb9ad7e26521a3b89db2", + "5a07101d8884c3f86496450121a048660f511b33076205395149ef6c369ef719", + "FILE:@@gazelle++non_module_deps+bazel_gazelle_go_repository_cache//go.env REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_tools\n"), + ("07b34597f53e981474b3e4952e51c5caa2ba3877f61ad0cc2fe39c84f51acc8f", + "f7eac27fd1a860cc4563bf18f42f0672d533e171341e7293aae8b5f5d8cdebc3", + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/fetch_repo ENV:GO_REPOSITORY_EPHEMERAL_MODCACHE\n"), + ("1ef1aa4b4cb73c42f39465f8feba3ae846c412e5a0a7f2790264264c1941df6e", + "531c6579bec5634c2744fdd5b90ea123739d431991472877bffe3548c86697b1", + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/gazelle\n"), + ("edc467b59551f6058310c401062c51315e9a1c3b2bc8ec0a57d7411d7ca4b6dc", + "94a6fc01bd63b6c1506919e676603784eb73e9049b121c4580aaf6e75893d276", + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//WORKSPACE\n"), +] # fmt: skip +POINTER_FINAL_HASH = "3ceee7642907f9b23db2d1944d05314384151ca509d432eca651bdee2d61997c" +POINTER_FINAL_KEY = v.Digest("b0ca33d505c031b1ee5aa17138c14135803b0989ef76f3203999a316a933e2f2", 206) + + +# --- an in-memory cache --------------------------------------------------------- + + +def enc_directory(files=(), dirs=(), symlinks=()): + out = b"" + for name, dg, exe in sorted(files): + node = v._field(1, name.encode()) + v._field(2, dg.encode()) + if exe: + node += v._field_varint(4, 1) + out += v._field(1, node) + for name, dg in sorted(dirs): + out += v._field(2, v._field(1, name.encode()) + v._field(2, dg.encode())) + for name, target in sorted(symlinks): + out += v._field(3, v._field(1, name.encode()) + v._field(2, target.encode())) + return out + + +class FakeCache: + """An AC (input hash -> ActionResult bytes) and a CAS, behind Cache's rpc.""" + + def __init__(self): + self.ac, self.cas = {}, {} + + def put_blob(self, data): + d = v.Digest.of(data) + self.cas[d.hash] = data + return d + + def tree_of(self, path): + """Uploads path's files and returns the Tree digest (as Bazel would).""" + children = [] + + def directory(p): + files, dirs, links = [], [], [] + for n in sorted(os.listdir(p)): + q = os.path.join(p, n) + st = os.lstat(q) + if stat.S_ISLNK(st.st_mode): + links.append((n, os.readlink(q))) + elif stat.S_ISDIR(st.st_mode): + raw = directory(q) + children.append(raw) + dirs.append((n, v.Digest.of(raw))) + else: + data = v.read_bytes(q) + files.append((n, self.put_blob(data), bool(st.st_mode & stat.S_IXUSR))) + return enc_directory(files, dirs, links) + + root = directory(path) + tree = v._field(1, root) + b"".join(v._field(2, c) for c in children) + return self.put_blob(tree) + + def put_final(self, input_hash, marker, tree, exit_code=0): + md = self.put_blob(marker) + ar = ( + v._field(2, v._field(1, v.MARKER_FILE_PATH.encode()) + v._field(2, md.encode())) + + v._field(3, v._field(1, v.REPO_DIRECTORY_PATH.encode()) + v._field(3, tree.encode())) + ) + if exit_code: + ar += v._field_varint(4, exit_code) + self.ac[input_hash] = ar + + def put_intermediate(self, input_hash, stdout, raw=True): + if raw: + self.ac[input_hash] = v._field(5, stdout.encode()) + v._field(6, v.Digest.of(stdout.encode()).encode()) + else: + self.ac[input_hash] = v._field(6, self.put_blob(stdout.encode()).encode()) + + def rpc(self, method, request): + if method.endswith("ActionCache/GetActionResult"): + key = None + for num, _, val in v.fields(request): + if num == 2: + key = v.Digest.decode(val) + for h, ar in self.ac.items(): + if v.action_digest(GUID, h) == key: + return [ar] + raise v.RPCError(v.GRPC_NOT_FOUND, "not found") + if method.endswith("ByteStream/Read"): + name = bytes(next(val for num, _, val in v.fields(request) if num == 1)).decode() + _, _, h, size = name.split("/") + data = self.cas.get(h) + if data is None: + raise v.RPCError(v.GRPC_NOT_FOUND, "no blob") + return [v._field(10, data[: len(data) // 2]), v._field(10, data[len(data) // 2 :])] + raise AssertionError(method) + + +def write(path, data, mode=0o644): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(data) + os.chmod(path, mode) + + +class Fixture: + """A cold output base with one repository, cached as fetched.""" + + def __init__(self, test, name="repo", marker=b"h0\n"): + self.tmp = tempfile.TemporaryDirectory() + test.addCleanup(self.tmp.cleanup) + self.ext = os.path.join(self.tmp.name, "external") + self.name = name + self.dir = os.path.join(self.ext, name) + write(os.path.join(self.dir, "BUILD.bazel"), b"filegroup(name = 'x')\n") + write(os.path.join(self.dir, "bin", "tool"), b"#!/bin/sh\n", 0o755) + write(os.path.join(self.dir, "sub", "deep", "data.txt"), b"data\n") + os.makedirs(os.path.join(self.dir, "empty")) + os.symlink("bin/tool", os.path.join(self.dir, "link")) + self.marker = marker + write(os.path.join(self.ext, "@%s.marker" % name), marker) + self.cache = FakeCache() + + def seed(self, input_hash="h0"): + self.cache.put_final(input_hash, self.marker, self.cache.tree_of(self.dir)) + + def verify(self): + return v.verify_repo(v.Cache(self.cache.rpc, "oss"), GUID, self.ext, self.name) + + +class KeysTest(unittest.TestCase): + def test_action_digest_matches_bazel_9_3_0(self): + for h, key, _ in POINTER_CHAIN: + self.assertEqual(v.action_digest(GUID, h).hash, key) + self.assertEqual(v.action_digest(GUID, POINTER_FINAL_HASH), POINTER_FINAL_KEY) + + def test_roll_forward_matches_bazel_9_3_0(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + path = os.path.join(tmp.name, "m") + write(path, POINTER_MARKER.encode()) + predeclared, recorded = v.read_marker(path) + self.assertEqual(predeclared, POINTER_CHAIN[0][0]) + h = predeclared + for i, (want, _, stdout) in enumerate(POINTER_CHAIN): + self.assertEqual(h, want, "hop %d" % i) + for inp in stdout.strip().split(" "): + h = v.roll_forward(h, recorded[inp]) + self.assertEqual(h, POINTER_FINAL_HASH) + + def test_guid_for_every_release_is_a_uuid(self): + for release, guid in v.BAZEL_RRC_GUIDS.items(): + self.assertRegex(guid, r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", release) + + def test_bazelversion_has_a_guid(self): + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(v.__file__)))) + path = os.path.join(root, ".bazelversion") + version = v.read_bytes(path).decode().strip() + self.assertIn(version, v.BAZEL_RRC_GUIDS, "a .bazelversion bump must add RemoteRepoContentsCacheImpl.GUID") + + +class VerifyRepoTest(unittest.TestCase): + def test_match(self): + f = Fixture(self) + f.seed() + r = f.verify() + self.assertEqual((r.status, r.detail), ("match", [])) + + def test_not_cached(self): + f = Fixture(self) + self.assertEqual(f.verify().status, "not_cached") + + def test_content_poisoned(self): + f = Fixture(self) + f.seed() + write(os.path.join(f.dir, "sub", "deep", "data.txt"), b"other\n") + r = f.verify() + self.assertEqual(r.status, "mismatch") + self.assertIn("sub/deep/data.txt: content differs", " ".join(r.detail)) + + def test_extra_and_missing_files(self): + f = Fixture(self) + f.seed() + write(os.path.join(f.dir, "init.go"), b"package x\n") + os.remove(os.path.join(f.dir, "BUILD.bazel")) + detail = " ".join(f.verify().detail) + self.assertIn("init.go: only in the cold fetch", detail) + self.assertIn("BUILD.bazel: only in the cache", detail) + + def test_executable_bit(self): + f = Fixture(self) + f.seed() + os.chmod(os.path.join(f.dir, "bin", "tool"), 0o644) + self.assertIn("bin/tool: executable bit differs", " ".join(f.verify().detail)) + + def test_symlink_target(self): + f = Fixture(self) + f.seed() + os.remove(os.path.join(f.dir, "link")) + os.symlink("sub", os.path.join(f.dir, "link")) + self.assertIn("link: symlink to", " ".join(f.verify().detail)) + + def test_empty_directory(self): + f = Fixture(self) + f.seed() + os.rmdir(os.path.join(f.dir, "empty")) + self.assertIn("empty: only in the cache", " ".join(f.verify().detail)) + + def test_marker_differs(self): + f = Fixture(self) + f.seed() + write(os.path.join(f.ext, "@repo.marker"), b"h0\nENV:X 1\n") + r = f.verify() + self.assertEqual(r.status, "mismatch") + self.assertIn("marker file differs", " ".join(r.detail)) + + def test_intermediate_chain(self): + marker = b"p0\nENV:A 1\nFILE:@@x//y abc\nENV:B \\0\n" + f = Fixture(self, marker=marker) + h1 = v.roll_forward(v.roll_forward("p0", "ENV:A 1"), "FILE:@@x//y abc") + h2 = v.roll_forward(h1, "ENV:B \\0") + f.cache.put_intermediate("p0", "ENV:Z\nENV:A FILE:@@x//y\n") # the first batch is another fetch's + f.cache.put_intermediate(h1, "ENV:B\n", raw=False) + f.seed(h2) + r = f.verify() + self.assertEqual((r.status, r.detail), ("match", [])) + + def test_intermediate_without_a_matching_batch(self): + f = Fixture(self, marker=b"p0\nENV:A 1\n") + f.cache.put_intermediate("p0", "ENV:Q\n") + f.seed(v.roll_forward("p0", "ENV:A 2")) # another value of A: not this fetch's + self.assertEqual(f.verify().status, "not_cached") + + def test_invalid_exit_code(self): + f = Fixture(self) + f.cache.put_final("h0", f.marker, f.cache.tree_of(f.dir), exit_code=1) + self.assertEqual(f.verify().status, "invalid") + + def test_invalid_shape(self): + f = Fixture(self) + f.cache.ac["h0"] = v._field_varint(4, 0) + v._field(7, b"stderr") + self.assertEqual(f.verify().status, "invalid") + + def test_tampered_blob_is_an_error(self): + f = Fixture(self) + f.seed() + tree = next(d for d in f.cache.cas if f.cache.cas[d].startswith(b"\x0a")) + f.cache.cas[tree] = f.cache.cas[tree] + b"\x00" + self.assertEqual(f.verify().status, "error") + + +class ReportTest(unittest.TestCase): + def test_exit_status(self): + R = v.Result + self.assertEqual(v.exit_status([R("a", "match"), R("b", "not_cached")], 0.01), 0) + self.assertEqual(v.exit_status([R("a", "match"), R("b", "mismatch")], 0.01), 1) + self.assertEqual(v.exit_status([R("a", "invalid")], 0.01), 1) + self.assertEqual(v.exit_status([R("a", "error")] + [R(str(i), "match") for i in range(200)], 0.01), 0) + self.assertEqual(v.exit_status([R("a", "error"), R("b", "match")], 0.01), 2) + self.assertEqual(v.exit_status([], 0.01), 2) + + def test_repos_lists_marked_repositories(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + ext = os.path.join(tmp.name, "external") + for n in ("a", "b+c"): + os.makedirs(os.path.join(ext, n)) + write(os.path.join(ext, "@%s.marker" % n), b"h\n") + os.makedirs(os.path.join(ext, "unmarked")) + write(os.path.join(ext, "@gone.marker"), b"h\n") + self.assertEqual(v.repos(ext), ["a", "b+c"]) + + def test_unknown_release_is_a_usage_error(self): + self.assertEqual(v.main(["--output-base", "/x", "--endpoint", "grpc://x:1", "--bazel-version", "0.0.0"]), 2) + + +if __name__ == "__main__": + unittest.main() From 9f5028c78ebc6d9389197bf00c30c37a96615ff4 Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 10 Oct 2026 02:50:41 +0000 Subject: [PATCH 031/151] test(show): run --comments-tail through every show route; CHANGELOG bullet under Added (#6618 follow-ups) (#7383) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from the #6618 adoption review, neither addressed there: - bd-6gl9h7 (Test Evidence Quality): no committed test ran `bd show` with --comments-tail set. show_comments_tail_test.go calls printComments and renderWatchComments directly, so passing 0 at any of the three call sites (show.go direct, show_display.go --watch, show_proxied_server.go / show_proxied_watch.go proxied) still passed the suite. TestEmbeddedShowCommentsTail (direct, direct --watch) and TestProxiedServerShowCommentsTail (proxied, proxied --watch) now run the built binary through each route over the same three comments with --comments-tail 1, and check the elision line, the surviving newest comment, the absence of both hidden ones, the off state (0 and a cap above the count) byte-identical to the flag's absence, the negative usage error, and --json --include-comments untouched. Mutating each call site to pass 0 fails the matching route tests (direct: direct_tail_1 + watch_tail_1; proxied: proxied_tail_1 + proxied_watch_tail_1) and nothing else. The off-state comparison strips the "💡 Tip:" line maybeShowTip may append after a direct text render (a per-run probability roll behind a per-tip frequency gate, independent of the flag); without that, the first render of a fresh store can carry the tip and the comparison fails on it. The embedded half lives in a *_embedded_test.go file because gen_embedded_cmd_shard_manifest.py discovers TestEmbedded* only there; both tests are registered in their shard manifests (--write, then --check rc 0 at 50 and 30) and in cmd/bd's go_test srcs. - bd-l6f1r7 (Release Safety): the CHANGELOG bullet for the flag sat under "### Fixed" in [Unreleased]; it is now the first bullet under the section's "### Added". Ran (GOTOOLCHAIN=go1.26.7): go vet ./cmd/bd/; gofmt -l; go test ./scripts/ -run 'Shard|Manifest|Coverage'; embedded leg (BEADS_TEST_EMBEDDED_DOLT=1) -run TestEmbeddedShowCommentsTail -count=3: 15/15 subtests PASS; proxied leg (BEADS_TEST_PROXIED_SERVER=1, BEADS_TEST_DOLT_SERVER=local, dolt 2.2.0, -tags gms_pure_go) -run TestProxiedServerShowCommentsTail: 5/5 PASS, 0 SKIP. Co-authored-by: Claude Fable 5.1 --- .github/scripts/embedded-cmd-test-shards.txt | 1 + .github/scripts/proxied-cmd-test-shards.txt | 1 + CHANGELOG.md | 18 +- cmd/bd/BUILD.bazel | 2 + cmd/bd/show_comments_tail_embedded_test.go | 69 ++++++ ..._comments_tail_proxied_integration_test.go | 208 ++++++++++++++++++ 6 files changed, 290 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/show_comments_tail_embedded_test.go create mode 100644 cmd/bd/show_comments_tail_proxied_integration_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index e6987e1f4..19022d08d 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -313,6 +313,7 @@ 100 65 TestEmbeddedSupersede 100 66 TestEmbeddedCloseBasicIsolatedStores +100 66 TestEmbeddedShowCommentsTail 100 67 TestEmbeddedAdminCompactReadOnlyAllowed 100 67 TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 72e73e898..97ff8d2cd 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -335,6 +335,7 @@ 34 12 TestProxiedServerReadyB 34 12 TestProxiedServerReadyWisp 34 12 TestProxiedServerReclaim +34 12 TestProxiedServerShowCommentsTail 34 12 TestProxiedServerUnclaim 34 13 TestProxiedServerCompactDolt diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b52a5b5a..20024c147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `dolt.auto-commit` the way `bd prune` does, as `bd: reclaim N expired lease(s)` on both routes. Over a connected HTTP workspace `bd reclaim` now works instead of refusing. +- **`bd show --comments-tail N`** renders only the last N comments in text + output (including under `--watch`) + ([#6618](https://github.com/gastownhall/beads/pull/6618)), preceded by one + elision line naming how many older ones were hidden. A render-only cap for + fat, append-only beads whose full comment history is hundreds of KB — + description and metadata are unchanged, and omitting the flag (or passing + `0`) is byte-identical to today's output. JSON output is untouched; + `--include-comments` still streams every comment there. + - `bd create --graph` now plans its batch through `issueops.BatchApplier` instead of the old `buildDomainGraphPlan` path, so a graph create gets the same atomic multi-row semantics as `bd batch apply`. A `waits-for` edge's @@ -1351,15 +1360,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 open, including the library API, `bd doctor --fix` and `bd bootstrap`, keeps the existing message. -- **`bd show --comments-tail N`** renders only the last N comments in text - output (including under `--watch`) - ([#6618](https://github.com/gastownhall/beads/pull/6618)), preceded by one - elision line naming how many older ones were hidden. A render-only cap for - fat, append-only beads whose full comment history is hundreds of KB — - description and metadata are unchanged, and omitting the flag (or passing - `0`) is byte-identical to today's output. JSON output is untouched; - `--include-comments` still streams every comment there. - ### Changed - **The managed shared Dolt server now opens its configured remotesapi diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 7d1742cef..eb8aa6cd2 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1124,6 +1124,8 @@ go_test( "shared_store_migrate_gate_test.go", "ship_embedded_test.go", "show_brief_deps_test.go", + "show_comments_tail_embedded_test.go", + "show_comments_tail_proxied_integration_test.go", "show_comments_tail_test.go", "show_current_test.go", "show_details_json_test.go", diff --git a/cmd/bd/show_comments_tail_embedded_test.go b/cmd/bd/show_comments_tail_embedded_test.go new file mode 100644 index 000000000..8b5dafd4a --- /dev/null +++ b/cmd/bd/show_comments_tail_embedded_test.go @@ -0,0 +1,69 @@ +//go:build cgo + +package main + +import ( + "os" + "strings" + "testing" +) + +// Direct-route half of the --comments-tail route tests; fixtures and the +// proxied half are in show_comments_tail_proxied_integration_test.go. + +// TestEmbeddedShowCommentsTail drives the direct route (show.go) and the +// direct --watch route (show_display.go) through the built binary. +func TestEmbeddedShowCommentsTail(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "ct") + issue := bdCreate(t, bd, dir, "Tail me", "--type", "task") + for _, text := range commentsTailTexts { + if out, err := bdRunWithFlockRetry(t, bd, dir, "comments", "add", issue.ID, text); err != nil { + t.Fatalf("bd comments add %q failed: %v\n%s", text, err, out) + } + } + + t.Run("direct_tail_1", func(t *testing.T) { + out := bdShowRaw(t, bd, dir, issue.ID, "--comments-tail", "1") + assertCommentsTailOne(t, "direct", out, issue.ID) + }) + + t.Run("direct_off_state_matches_absence", func(t *testing.T) { + without := bdShowRaw(t, bd, dir, issue.ID) + assertCommentsTailOff(t, "direct --comments-tail 0", bdShowRaw(t, bd, dir, issue.ID, "--comments-tail", "0"), without) + assertCommentsTailOff(t, "direct --comments-tail 5", bdShowRaw(t, bd, dir, issue.ID, "--comments-tail", "5"), without) + }) + + t.Run("direct_negative_is_usage_error", func(t *testing.T) { + out := bdShowFail2(t, bd, dir, issue.ID, "--comments-tail", "-1") + if !strings.Contains(out, "--comments-tail") || !strings.Contains(out, "non-negative") { + t.Errorf("negative cap did not fail as a usage error:\n%s", out) + } + }) + + t.Run("direct_json_untouched", func(t *testing.T) { + withFlag, err := bdRunWithFlockRetry(t, bd, dir, "show", issue.ID, "--json", "--include-comments", "--comments-tail", "1") + if err != nil { + t.Fatalf("bd show --json --comments-tail 1 failed: %v\n%s", err, withFlag) + } + without, err := bdRunWithFlockRetry(t, bd, dir, "show", issue.ID, "--json", "--include-comments") + if err != nil { + t.Fatalf("bd show --json failed: %v\n%s", err, without) + } + if string(withFlag) != string(without) { + t.Errorf("--json output changed under --comments-tail 1\n--- with flag\n%s\n--- without\n%s", withFlag, without) + } + if !strings.Contains(string(withFlag), commentsTailOldest) { + t.Errorf("--json --include-comments dropped the oldest comment under --comments-tail 1:\n%s", withFlag) + } + }) + + t.Run("watch_tail_1", func(t *testing.T) { + out := runShowWatchOnce(t, bd, dir, bdEnv(dir), issue.ID, "--watch", "--comments-tail", "1") + assertCommentsTailOne(t, "direct --watch", out, issue.ID) + }) +} diff --git a/cmd/bd/show_comments_tail_proxied_integration_test.go b/cmd/bd/show_comments_tail_proxied_integration_test.go new file mode 100644 index 000000000..ea56a1677 --- /dev/null +++ b/cmd/bd/show_comments_tail_proxied_integration_test.go @@ -0,0 +1,208 @@ +//go:build cgo + +package main + +import ( + "fmt" + "os/exec" + "strings" + "syscall" + "testing" + "time" +) + +// Route tests for `bd show --comments-tail N` (#6618 follow-up, bd-6gl9h7): +// the shared fixtures and the proxied routes live here; the direct routes +// are in show_comments_tail_embedded_test.go, which the embedded cmd shard +// manifest discovers by its *_embedded_test.go name. +// +// show_comments_tail_test.go pins printComments and renderWatchComments by +// calling them directly, so a regression that passed 0 at any of the three +// call sites — show.go (direct), show_display.go (direct --watch), +// show_proxied_server.go / show_proxied_watch.go (proxied, plain and +// --watch) — would still pass that file. These tests run the built binary +// through every route with the flag set, over the same three comments, and +// read the rendered text: the elision line names the hidden count and the +// issue, only the newest comment survives the cap, and the flag's off state +// (0, or a cap above the count) is byte-identical to its absence. + +const ( + commentsTailOldest = "CT-OLDEST-ALPHA" + commentsTailOlder = "CT-OLDER-BRAVO" + commentsTailNewest = "CT-NEWEST-CHARLIE" +) + +var commentsTailTexts = []string{commentsTailOldest, commentsTailOlder, commentsTailNewest} + +// commentsTailElision is the one muted line printComments prints in front of +// a capped render; the text is pinned here so a reworded line fails every +// route at once rather than one route drifting. +func commentsTailElision(hidden int, issueID string) string { + noun := "comments" + if hidden == 1 { + noun = "comment" + } + return fmt.Sprintf("… %d older %s hidden — bd show %s for the full record", hidden, noun, issueID) +} + +// assertCommentsTailOne checks a text render of the three-comment issue +// under --comments-tail 1: the elision line, the newest comment, and neither +// of the two hidden ones. +func assertCommentsTailOne(t *testing.T, route, out, issueID string) { + t.Helper() + if want := commentsTailElision(2, issueID); !strings.Contains(out, want) { + t.Errorf("%s: elision line missing, want %q in:\n%s", route, want, out) + } + if !strings.Contains(out, commentsTailNewest) { + t.Errorf("%s: newest comment %q missing from the capped render:\n%s", route, commentsTailNewest, out) + } + for _, hidden := range []string{commentsTailOldest, commentsTailOlder} { + if strings.Contains(out, hidden) { + t.Errorf("%s: hidden comment %q rendered under --comments-tail 1:\n%s", route, hidden, out) + } + } +} + +// stripShowTips removes the "💡 Tip: …" line maybeShowTip (show.go, on the +// direct route after a text render) may append. Whether a tip prints is a +// per-run probability roll behind a per-tip frequency gate recorded in the +// store, independent of --comments-tail, so of two renders that are +// otherwise identical one can carry the tip and the other not. Trailing +// newlines are normalized with it, since the tip block arrives as a blank +// line plus the tip line. +func stripShowTips(out string) string { + var kept []string + for _, line := range strings.Split(out, "\n") { + if strings.HasPrefix(line, "💡 Tip: ") { + continue + } + kept = append(kept, line) + } + return strings.TrimRight(strings.Join(kept, "\n"), "\n") + "\n" +} + +// assertCommentsTailOff checks that a render with the flag in its off state +// (0, or a cap at or above the count) is byte-identical to a render without +// the flag (tips stripped, see stripShowTips), and that the uncapped render +// carries every comment and no elision line. +func assertCommentsTailOff(t *testing.T, route, withFlag, without string) { + t.Helper() + withFlag, without = stripShowTips(withFlag), stripShowTips(without) + if withFlag != without { + t.Errorf("%s: render differs from the flag's absence\n--- with flag\n%s\n--- without\n%s", route, withFlag, without) + } + for _, text := range commentsTailTexts { + if !strings.Contains(without, text) { + t.Errorf("%s: uncapped render lacks comment %q:\n%s", route, text, without) + } + } + if strings.Contains(without, "older comment") { + t.Errorf("%s: uncapped render carries an elision line:\n%s", route, without) + } +} + +// runShowWatchOnce starts `bd show ` (the caller passes --watch), waits +// for the first render to land, stops the watch with SIGINT and returns the +// stdout of that single render. The render/quiet/stop choreography is the +// one TestProxiedServerShowWatch uses, so see its comments for why the +// banner alone is not proof that stdout has fully landed. +func runShowWatchOnce(t *testing.T, bd, dir string, env []string, args ...string) string { + t.Helper() + stdout := &watchBannerWriter{renders: make(chan int, 1)} + stderr := &watchBannerWriter{renders: make(chan int, 1)} + cmd := exec.Command(bd, append([]string{"show"}, args...)...) + cmd.Dir = dir + cmd.Env = env + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start bd show %s: %v", strings.Join(args, " "), err) + } + var waitErr error + exited := make(chan struct{}) + go func() { + waitErr = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + if !stderr.waitForRender(1, 60*time.Second, exited) { + t.Fatalf("bd show %s never started watching\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), stdout.String(), stderr.String()) + } + stdout.waitForQuiet(500*time.Millisecond, 10*time.Second, exited) + if err := cmd.Process.Signal(syscall.SIGINT); err != nil { + t.Fatalf("signal bd show --watch: %v", err) + } + select { + case <-exited: + if waitErr != nil { + t.Fatalf("bd show %s exited with %v after SIGINT, want 0\nstderr:\n%s", strings.Join(args, " "), waitErr, stderr.String()) + } + case <-time.After(30 * time.Second): + t.Fatalf("bd show %s ignored SIGINT\nstderr:\n%s", strings.Join(args, " "), stderr.String()) + } + return stdout.String() +} + +// TestProxiedServerShowCommentsTail drives the proxied route +// (show_proxied_server.go) and the proxied --watch route +// (show_proxied_watch.go) through the built binary against the shared +// proxied server. +func TestProxiedServerShowCommentsTail(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "pct") + issue := bdProxiedCreate(t, bd, p.dir, "Tail me", "--type", "task") + for _, text := range commentsTailTexts { + bdProxiedComment(t, bd, p.dir, issue.ID, text) + } + + t.Run("proxied_tail_1", func(t *testing.T) { + t.Parallel() + out := bdProxiedShowRaw(t, bd, p.dir, issue.ID, "--comments-tail", "1") + assertCommentsTailOne(t, "proxied", out, issue.ID) + }) + + t.Run("proxied_off_state_matches_absence", func(t *testing.T) { + t.Parallel() + without := bdProxiedShowRaw(t, bd, p.dir, issue.ID) + assertCommentsTailOff(t, "proxied --comments-tail 0", bdProxiedShowRaw(t, bd, p.dir, issue.ID, "--comments-tail", "0"), without) + assertCommentsTailOff(t, "proxied --comments-tail 5", bdProxiedShowRaw(t, bd, p.dir, issue.ID, "--comments-tail", "5"), without) + }) + + t.Run("proxied_negative_is_usage_error", func(t *testing.T) { + t.Parallel() + stdout, stderr := bdProxiedShowFail(t, bd, p.dir, issue.ID, "--comments-tail", "-1") + combined := stdout + stderr + if !strings.Contains(combined, "--comments-tail") || !strings.Contains(combined, "non-negative") { + t.Errorf("negative cap did not fail as a usage error\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + }) + + t.Run("proxied_json_untouched", func(t *testing.T) { + t.Parallel() + withFlag, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "show", issue.ID, "--json", "--include-comments", "--comments-tail", "1") + if err != nil { + t.Fatalf("bd show --json --comments-tail 1 failed: %v\n%s", err, stderr) + } + without, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "show", issue.ID, "--json", "--include-comments") + if err != nil { + t.Fatalf("bd show --json failed: %v\n%s", err, stderr) + } + if withFlag != without { + t.Errorf("--json output changed under --comments-tail 1\n--- with flag\n%s\n--- without\n%s", withFlag, without) + } + if !strings.Contains(withFlag, commentsTailOldest) { + t.Errorf("--json --include-comments dropped the oldest comment under --comments-tail 1:\n%s", withFlag) + } + }) + + t.Run("proxied_watch_tail_1", func(t *testing.T) { + t.Parallel() + out := runShowWatchOnce(t, bd, p.dir, bdProxiedEnv(p.dir), issue.ID, "--watch", "--comments-tail", "1") + assertCommentsTailOne(t, "proxied --watch", out, issue.ID) + }) +} From 11b5d3271ecf868c3c0dcc273cf4d471c10a9b54 Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 10 Oct 2026 05:28:43 +0000 Subject: [PATCH 032/151] docs(proxied): name the Close3/Close4 legacy pins and fix the Close3 doc comment (#7173 follow-up) (#7382) * docs(proxied): name the Close3/Close4 legacy pins and fix the Close3 doc comment (#7173 follow-up) Two non-blocking findings from the #7173 adoption review (bd-vrdrew, bd-dars9p), both in bee's code, neither addressed in that PR: - The doc comment on TestProxiedServerClose3 still called it "the second half of TestProxiedServerClose"; after the second split it is the middle of the three parents (Close, Close3, Close4). Say so, and name Close4 as its own second half. - In the frozen 15-shard block of .github/scripts/proxied-cmd-test-shards.txt the lines `15 13 TestProxiedServerClose3` and `15 2 TestProxiedServerClose4` sat inside the shard-1 and shard-7 groups (#7173's last commit moved the pins from 1 and 7 to 13 and 2 without moving the lines), and the header's "Deliberate exception (Oct 2026)" paragraph did not name them. The two lines now sit alphabetically inside their own shard groups, the header names them with the reason and the measured times, and the generator's hand-pin paragraph (scripts/ci/gen_proxied_shard_manifest.py) names them too so the two texts stay in step. The lines were inert either way: proxied-test-shard.sh matches each line by field. No test moves shard: the 15-shard block's line set is unchanged apart from order (sorted diff empty) and the 30-shard block is byte-identical. Verified: gofmt -l (clean); go vet ./cmd/bd/ (exit 0); gen_proxied_shard_manifest.py 30 --weights=duration --check (exit 0, heaviest 435.1s est, unchanged); 15 --check reports the same 30 hashed names missing as main (the frozen block is expected to fail it); proxied-test-shard.sh list-only: Close3 on 13/15, Close4 on 2/15, neither on 1/15 or 7/15; go test ./scripts/ -run 'Shard|Manifest|Coverage' ok 38.6s. Co-Authored-By: Claude Fable 5.1 * docs(proxied): count CloseB in the Close3 comment; say why shard 5 got no pin Maintainer fixup from the adoption review of #7382 (bd-g6pyb0), on top of bee's commit, which stays as written. Comment and manifest-header text only: no test body, shard assignment or routing changes. - TestProxiedServerClose3's doc comment called it "the middle of the three parents TestProxiedServerClose was split into (Close, Close3, Close4)". That held at the PR's original base, but #7418, on main since, moved five of Close's eleven cases into a new TestProxiedServerCloseB, so the 23 bd-init subtests of the pre-#7173 TestProxiedServerClose now sit in four parents: Close 6, CloseB 5, Close3 6, Close4 6 (their union is exactly the pre-#7173 set, no duplicates). The comment now says Close3 was split off Close, names all four parents, and keeps "Its own second half is TestProxiedServerClose4." - The frozen 15-shard block's header said #7173 pinned Close3 and Close4 onto "the two lightest shards of that PR's own run (6m51s and 7m57s)". On that run (37206446441, at 765248af5) shard 5 was lighter still, at 5m50s; #7173 passed over it on purpose, because it was light only because those 12 subtests had just left it and putting Close3 back would recreate most of the set that timed out. The header now says "other than shard 5 itself" and gives that reason. This commit also supersedes the parts of the parent commit's message that the rebase onto c21880105f outdated: - "after the second split it is the middle of the three parents (Close, Close3, Close4)": four parents now, as above. - "the 30-shard block is byte-identical": the Bazel-only block is 34 shards now (#7418 made it 44, #7436 34), and that block is byte-identical to the base's. - "gen_proxied_shard_manifest.py 30 --weights=duration --check (exit 0, heaviest 435.1s est, unchanged)": with no 30-shard block left, that command now exits 1 ("has no block for total_shards=30"). Its current form, 34 --weights=duration --check, exits 0 with the heaviest shard at 196.1 s est., the same before and after this commit. - "15 --check reports the same 30 hashed names missing as main": it now reports 53, the same set as against the base's manifest. Verified: git diff --check clean; gofmt -l clean (the Go change is comment-only); with '#' lines stripped as proxied-test-shard.sh strips them, the manifest is identical before and after; gen_proxied_shard_manifest.py 15 --check and 34 --weights=duration --check print the same output before and after (exit 1 with 53 missing; exit 0, heaviest 196.1 s); BEADS_TEST_SHARD_LIST_ONLY=1 proxied-test-shard.sh lists for 1, 2, 7 and 13 of 15 are unchanged (Close3 on 13, Close4 on 2, neither on 1 or 7); go test ./scripts/ -run 'Shard|Manifest|Coverage' ok (21 top-level tests, none skipped). Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- .github/scripts/proxied-cmd-test-shards.txt | 13 +++++++++++-- cmd/bd/close_proxied_integration_test.go | 13 ++++++++----- scripts/ci/gen_proxied_shard_manifest.py | 4 +++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 97ff8d2cd..aaec871fd 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -34,6 +34,15 @@ # have no socat, so today it skips here; the pin keeps shard 14 safe if # socat is ever installed. # +# TestProxiedServerClose3 (shard 13) and TestProxiedServerClose4 (shard 2) +# are the two parents gastownhall/beads#7173 split off TestProxiedServerClose +# to pull shard 5 (15m+) back under the timeout, pinned onto the two +# lightest shards of that PR's own run other than shard 5 itself (6m51s +# and 7m57s; shard 5 ran 5m50s only because their 12 subtests had just +# left it, and putting Close3 back would recreate most of the set that +# timed out) rather than hashed; the lines are listed under their shards +# like every other pin. +# # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator instead. Known blind spots # of the cost proxy (it counts inits, not the work done per init): @@ -51,7 +60,6 @@ # a doc comment. 15 1 TestProxiedServerBlocked -15 13 TestProxiedServerClose3 15 1 TestProxiedServerCompactDolt 15 1 TestProxiedServerCreate 15 1 TestProxiedServerDep2 @@ -64,6 +72,7 @@ 15 1 TestProxiedServerServeRefusesAStreamOnADisabledJournal 15 1 TestProxiedServerUpdateConcurrentClaim +15 2 TestProxiedServerClose4 15 2 TestProxiedServerCompactHistory 15 2 TestProxiedServerCreate2 15 2 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge @@ -119,7 +128,6 @@ 15 7 TestProxiedServerAssign 15 7 TestProxiedServerBackupRefusedOnExternalTopology -15 2 TestProxiedServerClose4 15 7 TestProxiedServerCount 15 7 TestProxiedServerGC 15 7 TestProxiedServerList @@ -185,6 +193,7 @@ 15 12 TestProxiedServerTypes 15 12 TestProxiedServerUpdateClosePolicy +15 13 TestProxiedServerClose3 15 13 TestProxiedServerDeleteConcurrent 15 13 TestProxiedServerInfo 15 13 TestProxiedServerMolSquash diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index 0e349f740..94baa1ad6 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -358,11 +358,14 @@ func TestProxiedServerCloseB(t *testing.T) { }) } -// TestProxiedServerClose3 is the second half of TestProxiedServerClose, -// split off (as TestProxiedServerClose2 was before it) so that no single -// top-level suite carries 23 bd-init subtests: that one parent alone cost -// ~2060 slot-seconds under -test.parallel=4 and pushed its 15-shard legacy -// shard past go test's 15m timeout on every run (gastownhall/beads#7151). +// TestProxiedServerClose3 was split off TestProxiedServerClose (as +// TestProxiedServerClose2 was before it) so that no single top-level suite +// carries 23 bd-init subtests: that one parent alone cost ~2060 +// slot-seconds under -test.parallel=4 and pushed its 15-shard legacy shard +// past go test's 15m timeout on every run (gastownhall/beads#7151). Those +// 23 now sit in TestProxiedServerClose, TestProxiedServerCloseB, +// TestProxiedServerClose3 and TestProxiedServerClose4. Its own second half +// is TestProxiedServerClose4. func TestProxiedServerClose3(t *testing.T) { requireSharedProxiedServer(t) t.Parallel() diff --git a/scripts/ci/gen_proxied_shard_manifest.py b/scripts/ci/gen_proxied_shard_manifest.py index ecac80c89..a6f04a7db 100644 --- a/scripts/ci/gen_proxied_shard_manifest.py +++ b/scripts/ci/gen_proxied_shard_manifest.py @@ -222,7 +222,9 @@ def render(total, shards, weights): out.append('# TestProxiedServerListComments and') out.append('# TestProxiedServerServeRefusesAStreamOnADisabledJournal; and for') out.append('# TestProxiedServerOutageReconnectAcceptanceMatrix, pinned to shard 1') - out.append('# instead of hashing onto shard 14).') + out.append('# instead of hashing onto shard 14; and for TestProxiedServerClose3') + out.append('# and TestProxiedServerClose4, the parents #7173 split off') + out.append('# TestProxiedServerClose, pinned to shards 13 and 2).') else: out.append(f'# {total}-shard split for the Bazel-only proxied-server tier') out.append('# (bazel-proxied in .github/workflows/bazel.yml), bin-packed') From b3cdbf9e466f49ec2a2ae1030d72d964e15d8ee8 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 05:58:24 +0000 Subject: [PATCH 033/151] ci(rbe): auto-PR a worker-env pin sync when gascity main re-pins (mc-gv9fk) (#7455) rbe-west's pool workers advertise gastownhall/gascity main's worker-env pin; when gascity re-pins (a Go/dolt/OS bump, e.g. #7387) beads' copy falls out of step and every remote-execution run stops in bazel.yml's preflight until a human notices and copies the fix by hand (#7419 took ~100 minutes). This adds a scheduled workflow (every 15 minutes, plus workflow_dispatch) that runs tools/rbe/worker-env-sync against gascity main, and on a real mismatch (exit 1) copies the manifest and pin with the new tools/rbe/worker-env-sync-copy, opens/updates a PR on a fixed branch with status/needs-review-auto, and arms auto-merge. GitHub never runs checks on a PR a plain GITHUB_TOKEN opens, so this needs a GitHub App token -- but gastownhall-autofix (contents:write on this repo) cannot open or label a PR. Two Apps split the two write roles instead: gastownhall-autofix mints push-token (contents:write only) and does the git push of bot/worker-env-sync via plain git; gascity-autofix (pull-requests:write only) mints pr-token and does everything PR-shaped -- create/update, the label, and arming auto-merge -- via gh. peter-evans/create-pull-request takes one token for the whole flow, so it can't honor this split; plain git and gh replace it. Every ${{ }} is resolved into a step's env: first, never written inside a run: script. Both Apps' id/private-key secrets live in the same autofix environment, gated by its deployment-branch policy to main; schedule and workflow_dispatch are never fork events, so neither key is ever readable by a fork. Needs admin setup: the gascity-autofix App (id 5259233) installed on gastownhall/beads (and gascity) with pull-requests:write and metadata read, and its GASCITY_AUTOFIX_APP_ID / GASCITY_AUTOFIX_APP_PRIVATE_KEY secrets added to the autofix environment alongside the existing AUTOFIX_APP_ID / AUTOFIX_APP_PRIVATE_KEY. Co-authored-by: Claude Sonnet 5 --- .github/workflows/worker-env-autosync.yml | 196 ++++++++++++++++ scripts/BUILD.bazel | 1 + scripts/worker_env_autosync_test.go | 259 ++++++++++++++++++++++ tools/rbe/worker-env-sync-copy | 41 ++++ 4 files changed, 497 insertions(+) create mode 100644 .github/workflows/worker-env-autosync.yml create mode 100644 scripts/worker_env_autosync_test.go create mode 100755 tools/rbe/worker-env-sync-copy diff --git a/.github/workflows/worker-env-autosync.yml b/.github/workflows/worker-env-autosync.yml new file mode 100644 index 000000000..2978a5617 --- /dev/null +++ b/.github/workflows/worker-env-autosync.yml @@ -0,0 +1,196 @@ +# Keep beads' rbe worker-env pin in step with gastownhall/gascity main +# without the ~100-minute wait #7419 fixed by hand. +# +# rbe-west's OSS pool is gascity's: its workers advertise the sha256 of the +# toolchain manifest gascity's tools/rbe/worker-env measures, and the +# schedulers match //platforms:rbe_worker's worker-env exec property +# exactly. beads' tools/rbe/worker-env.txt and the platforms/BUILD.bazel pin +# are a byte-for-byte copy of gascity main's; when gascity moves the pin (a +# Go, dolt, or OS/library bump -- #7387 was a Go security release), every +# beads remote-execution run stops in bazel.yml's rbe-job preflight +# (tools/rbe/worker-env-sync) until the copy catches up. nightly.yml's +# rbe-worker-env-sync job already opens a ci/infra issue for a human to fix; +# this instead opens the fix itself, every 15 minutes, so the gap is +# minutes, not until someone reads the issue. +# +# tools/rbe/worker-env-sync only detects the mismatch (exit 1) or that it +# could not compare with gascity (exit 2, e.g. GitHub is briefly +# unreachable; the next run retries). The write half, tools/rbe/worker-env- +# sync-copy, runs only after that exit-1 detection, on a checkout already in +# hand -- so a transient gascity outage can turn "in step" into "unknown", +# never into an unwanted write. +# +# TWO TOKENS, ONE JOB EACH: GitHub never runs checks on a PR a plain +# GITHUB_TOKEN opens (its loop-prevention rule), so this needs a GitHub App +# token to open the PR. The gastownhall-autofix App (contents:write only on +# this repo) mints push-token and does the git push of the fixed branch +# (bot/worker-env-sync); it cannot open, label or merge a PR. The separate +# gascity-autofix App (pull-requests:write only, no contents) mints pr-token +# and does everything PR-shaped: create/update, the status/needs-review-auto +# label, and arming auto-merge. Splitting them means a bug in the PR-shaped +# half can never push a commit, and a bug in the push half can never touch a +# PR; neither token can do the other's job even by accident. Both Apps' +# credentials are secrets of the same `autofix` environment. +# +# peter-evans/create-pull-request takes one token for the whole push+PR +# flow, so it cannot honor this split; this uses plain git for the push and +# gh for the PR instead. Every `${{ }}` is resolved into a step's `env:`, +# never written inside a `run:` script, so a value that happened to look +# like shell (a title or body containing `` ` `` or `$`) is data to the +# shell, not something GitHub's expression syntax or the shell re-parses. +# +# SECURITY: schedule and workflow_dispatch are never fork events (no +# pull_request trigger here at all), so neither App's private key -- secrets +# of the `autofix` environment, whose deployment-branch policy allows only +# main -- is ever readable by a fork. The guard on github.repository is +# defense in depth for a fork of this repo, where the secrets would not +# exist anyway. Both mint steps run only once worker-env-sync has already +# found a real mismatch, so a run that is in step or can't compare asks +# neither App for a token. gh pr merge --auto only arms GitHub's native +# auto-merge (it still waits for every required check and the branch +# protection rule); it does not merge anything itself. +name: rbe worker-env autosync + +on: + schedule: + - cron: "*/15 * * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + autosync: + name: Sync worker-env pin with gascity main + if: github.repository == 'gastownhall/beads' + runs-on: ubuntu-latest + timeout-minutes: 10 + # Grants the job read access to this environment's secrets (both Apps' + # id and private key); its deployment-branch policy restricts that to + # jobs running on main, which schedule and workflow_dispatch on this + # repo always are. + environment: autofix + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false + + # Never fails the step itself: rc is the signal every later step + # gates on (1, a real mismatch; 2, gascity unreachable -- the next + # scheduled run retries; 0, in step). A run every 15 minutes failing + # red for however long a mismatch takes to review and merge would be + # noise, not signal. + - name: Compare the worker-env manifest and pin with gascity's main + id: sync + run: | + rc=0 + tools/rbe/worker-env-sync main || rc=$? + echo "rc=$rc" >> "$GITHUB_OUTPUT" + + - name: Copy gascity main's worker-env manifest and pin + if: steps.sync.outputs.rc == '1' + run: tools/rbe/worker-env-sync-copy main + + - name: Confirm only the worker-env files moved + if: steps.sync.outputs.rc == '1' + run: | + changed="$(git diff --name-only)" + untracked="$(git ls-files --others --exclude-standard)" + want=$'platforms/BUILD.bazel\ntools/rbe/worker-env.txt' + if [ -n "$untracked" ] || [ "$(printf '%s\n' "$changed" | sort)" != "$(printf '%s\n' "$want" | sort)" ]; then + echo "::error::worker-env-sync-copy touched more than platforms/BUILD.bazel and tools/rbe/worker-env.txt:" + printf '%s\n' "$changed" "$untracked" + exit 1 + fi + + # contents:write only, on this repo alone: all it can do is push + # bot/worker-env-sync. Minted only once a real mismatch needs a push. + - name: Mint push token (gastownhall-autofix) + id: push-token + if: steps.sync.outputs.rc == '1' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ secrets.AUTOFIX_APP_ID }} + private-key: ${{ secrets.AUTOFIX_APP_PRIVATE_KEY }} + owner: gastownhall + repositories: beads + permission-contents: write + + # pull-requests:write only, on this repo alone: all it can do is + # open, label and arm auto-merge on a PR -- never push a commit. + - name: Mint PR token (gascity-autofix) + id: pr-token + if: steps.sync.outputs.rc == '1' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ secrets.GASCITY_AUTOFIX_APP_ID }} + private-key: ${{ secrets.GASCITY_AUTOFIX_APP_PRIVATE_KEY }} + owner: gastownhall + repositories: beads + permission-pull-requests: write + + # push-token alone; it cannot see a PR exists, let alone touch one. + # Leased to the branch: a force-with-lease scoped to refs/heads/ + # bot/worker-env-sync only, so a push can never move any other ref. + - name: Commit and push the sync branch + if: steps.sync.outputs.rc == '1' + env: + PUSH_TOKEN: ${{ steps.push-token.outputs.token }} + REPO: ${{ github.repository }} + BRANCH: bot/worker-env-sync + COMMIT_SUBJECT: "build(rbe): sync worker-env pin with gascity main" + GIT_AUTHOR_NAME: gastownhall-autofix[bot] + GIT_AUTHOR_EMAIL: gastownhall-autofix[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: gastownhall-autofix[bot] + GIT_COMMITTER_EMAIL: gastownhall-autofix[bot]@users.noreply.github.com + run: | + git checkout -B "$BRANCH" + git add platforms/BUILD.bazel tools/rbe/worker-env.txt + git commit -m "$COMMIT_SUBJECT" + git push "https://x-access-token:${PUSH_TOKEN}@github.com/${REPO}.git" \ + "HEAD:refs/heads/$BRANCH" "--force-with-lease=refs/heads/$BRANCH" + + # pr-token alone; it cannot push a commit, only act on the PR the + # step above already pushed. + - name: Open or update the sync PR + id: pr + if: steps.sync.outputs.rc == '1' + env: + GH_TOKEN: ${{ steps.pr-token.outputs.token }} + REPO: ${{ github.repository }} + BRANCH: bot/worker-env-sync + TITLE: "build(rbe): sync worker-env pin with gascity main" + LABEL: status/needs-review-auto + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + body="$(cat <> "$GITHUB_OUTPUT" + + - name: Enable auto-merge + if: steps.sync.outputs.rc == '1' && steps.pr.outputs.number + env: + GH_TOKEN: ${{ steps.pr-token.outputs.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + run: gh pr merge --auto --squash --repo "$REPO" "$PR_NUMBER" diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 08cf03e48..b3d3294a1 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -61,6 +61,7 @@ go_test( "pr_risk_bazel_coverage_test.go", "rbe_worker_env_test.go", "repo_helpers_test.go", + "worker_env_autosync_test.go", ], # keep data = [ "//:repo_other_files", diff --git a/scripts/worker_env_autosync_test.go b/scripts/worker_env_autosync_test.go new file mode 100644 index 000000000..f3a4871f7 --- /dev/null +++ b/scripts/worker_env_autosync_test.go @@ -0,0 +1,259 @@ +package scripts_test + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// worker-env-autosync.yml closes the #7419 gap: nightly.yml's rbe-worker- +// env-sync job only opens a ci/infra issue for a human to fix a gascity +// re-pin (#7387's ~100-minute stall); this instead runs tools/rbe/worker- +// env-sync every 15 minutes and, on a real mismatch (exit 1), copies +// gascity main's manifest and pin (tools/rbe/worker-env-sync-copy) and +// opens or updates a fixed-branch PR with auto-merge armed. +const ( + workerEnvAutosyncWorkflow = "worker-env-autosync.yml" + workerEnvAutosyncJob = "autosync" + workerEnvAutosyncBranch = "bot/worker-env-sync" + workerEnvSyncCopy = "tools/rbe/worker-env-sync-copy" +) + +// TestWorkerEnvAutosyncWorkflowTriggersOnScheduleAndDispatchOnly: the App +// private key this job mints is a secret of the autofix environment +// (deployment-branch policy: main only), and the only events that can ever +// run on main without a fork in the loop are schedule and workflow_dispatch +// -- pull_request (or pull_request_target) here would hand a fork a path to +// it. +func TestWorkerEnvAutosyncWorkflowTriggersOnScheduleAndDispatchOnly(t *testing.T) { + var parsed struct { + On map[string]any `yaml:"on"` + } + text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+workerEnvAutosyncWorkflow) + if err := yaml.Unmarshal([]byte(text), &parsed); err != nil { + t.Fatal(err) + } + if got := len(parsed.On); got != 2 { + t.Fatalf("%s on: has %d triggers (%v), want exactly schedule and workflow_dispatch", workerEnvAutosyncWorkflow, got, parsed.On) + } + if _, ok := parsed.On["workflow_dispatch"]; !ok { + t.Errorf("%s has no workflow_dispatch trigger", workerEnvAutosyncWorkflow) + } + schedule, ok := parsed.On["schedule"].([]any) + if !ok || len(schedule) != 1 { + t.Fatalf("%s schedule = %+v, want exactly one cron entry", workerEnvAutosyncWorkflow, parsed.On["schedule"]) + } + cron, _ := schedule[0].(map[string]any) + if cron["cron"] != "*/15 * * * *" { + t.Errorf("%s schedule cron = %v, want */15 * * * * (every 15 minutes)", workerEnvAutosyncWorkflow, cron["cron"]) + } + for _, forbidden := range []string{"pull_request", "pull_request_target", "push", "merge_group", "workflow_run"} { + if _, ok := parsed.On[forbidden]; ok { + t.Errorf("%s must never run on %s (forks or an unprivileged push could reach the autofix App token)", workerEnvAutosyncWorkflow, forbidden) + } + } +} + +// TestWorkerEnvAutosyncWorkflowPermissionsAreMinimal: least privilege. +// Top-level and the job's own grant both stay read-only: every write this +// job makes (the push, the PR, the label, auto-merge) goes through one of +// the two minted App tokens, never the job's own GITHUB_TOKEN permissions, +// so there is nothing here for either App token's role to leak into. +func TestWorkerEnvAutosyncWorkflowPermissionsAreMinimal(t *testing.T) { + var top struct { + Permissions map[string]string `yaml:"permissions"` + } + text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+workerEnvAutosyncWorkflow) + if err := yaml.Unmarshal([]byte(text), &top); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(top.Permissions, map[string]string{"contents": "read"}) { + t.Errorf("%s top-level permissions = %v, want contents: read", workerEnvAutosyncWorkflow, top.Permissions) + } + + job := readCIWorkflow(t, workerEnvAutosyncWorkflow).job(t, workerEnvAutosyncJob) + wantJobPerms := map[string]any{"contents": "read"} + if !reflect.DeepEqual(job.Permissions, wantJobPerms) { + t.Errorf("%s job %s permissions = %v, want %v (contents:read alone; both write roles come from the minted App tokens)", workerEnvAutosyncWorkflow, workerEnvAutosyncJob, job.Permissions, wantJobPerms) + } + if job.RunsOn != "ubuntu-latest" { + t.Errorf("%s job %s runs-on = %q, want ubuntu-latest", workerEnvAutosyncWorkflow, workerEnvAutosyncJob, job.RunsOn) + } + if job.If != "github.repository == 'gastownhall/beads'" { + t.Errorf("%s job %s if = %q, want the fork guard github.repository == 'gastownhall/beads'", workerEnvAutosyncWorkflow, workerEnvAutosyncJob, job.If) + } + if job.Environment.Name != "autofix" { + t.Errorf("%s job %s environment = %q, want autofix (both Apps' id/private-key secrets live there, gated to main)", workerEnvAutosyncWorkflow, workerEnvAutosyncJob, job.Environment.Name) + } +} + +// TestWorkerEnvAutosyncWorkflowShape: the job compares with gascity main, +// gates every write step on a real mismatch (rc == 1, never rc == 2's +// "unknown" or rc == 0's "in step"), mints two App tokens only once a +// mismatch needs a write, pushes the fixed branch with push-token alone, +// opens/labels the PR and arms auto-merge with pr-token alone (so it is +// not a GITHUB_TOKEN PR - those never trigger CI), and arms auto-merge +// rather than merging directly. +func TestWorkerEnvAutosyncWorkflowShape(t *testing.T) { + job := readCIWorkflow(t, workerEnvAutosyncWorkflow).job(t, workerEnvAutosyncJob) + + sync := job.step(t, "Compare the worker-env manifest and pin with gascity's main") + if sync.ID != "sync" || !strings.Contains(sync.Run, "tools/rbe/worker-env-sync main") { + t.Errorf("sync step: id %q, run %q; want id sync running tools/rbe/worker-env-sync main", sync.ID, sync.Run) + } + + const mismatchIf = "steps.sync.outputs.rc == '1'" + for _, name := range []string{ + "Copy gascity main's worker-env manifest and pin", + "Confirm only the worker-env files moved", + "Mint push token (gastownhall-autofix)", + "Mint PR token (gascity-autofix)", + "Commit and push the sync branch", + "Open or update the sync PR", + } { + step := job.step(t, name) + if step.If != mismatchIf { + t.Errorf("step %q if = %q, want %q (a write step must run only on a real mismatch)", name, step.If, mismatchIf) + } + } + + copyStep := job.step(t, "Copy gascity main's worker-env manifest and pin") + if !strings.Contains(copyStep.Run, workerEnvSyncCopy+" main") { + t.Errorf("copy step run %q, want it to call %s main", copyStep.Run, workerEnvSyncCopy) + } + + pushMint := job.step(t, "Mint push token (gastownhall-autofix)") + if pushMint.ID != "push-token" || actionFamily(pushMint.Uses) != "actions/create-github-app-token" || + pushMint.With["app-id"] != "${{ secrets.AUTOFIX_APP_ID }}" || pushMint.With["private-key"] != "${{ secrets.AUTOFIX_APP_PRIVATE_KEY }}" || + pushMint.With["permission-contents"] != "write" || pushMint.With["permission-pull-requests"] != "" { + t.Errorf("push-token mint = %+v, want the gastownhall-autofix mint (AUTOFIX_APP_ID/AUTOFIX_APP_PRIVATE_KEY, contents:write only)", pushMint) + } + + prMint := job.step(t, "Mint PR token (gascity-autofix)") + if prMint.ID != "pr-token" || actionFamily(prMint.Uses) != "actions/create-github-app-token" || + prMint.With["app-id"] != "${{ secrets.GASCITY_AUTOFIX_APP_ID }}" || prMint.With["private-key"] != "${{ secrets.GASCITY_AUTOFIX_APP_PRIVATE_KEY }}" || + prMint.With["permission-pull-requests"] != "write" || prMint.With["permission-contents"] != "" { + t.Errorf("pr-token mint = %+v, want the gascity-autofix mint (GASCITY_AUTOFIX_APP_ID/GASCITY_AUTOFIX_APP_PRIVATE_KEY, pull-requests:write only)", prMint) + } + if pushMint.Uses != prMint.Uses { + t.Errorf("push-token and pr-token mint different action pins: %q vs %q, want the same SHA-pinned actions/create-github-app-token", pushMint.Uses, prMint.Uses) + } + + push := job.step(t, "Commit and push the sync branch") + if push.Uses != "" || !strings.Contains(push.Run, "git push") { + t.Errorf("push step = %+v, want a plain git push (no action; push-token and pr-token cannot be split through one action's single token input)", push) + } + if push.Env["PUSH_TOKEN"] != "${{ steps.push-token.outputs.token }}" { + t.Errorf("push step env PUSH_TOKEN = %q, want the minted push-token", push.Env["PUSH_TOKEN"]) + } + if _, leaked := push.Env["GH_TOKEN"]; leaked { + t.Errorf("push step sets GH_TOKEN; the push step must never see pr-token") + } + if !strings.Contains(push.Run, workerEnvAutosyncBranch) && push.Env["BRANCH"] != workerEnvAutosyncBranch { + t.Errorf("push step does not reference the fixed branch %q", workerEnvAutosyncBranch) + } + if !strings.Contains(push.Run, "force-with-lease=refs/heads/$BRANCH") { + t.Errorf("push step run %q, want a --force-with-lease scoped to the fixed branch's ref alone", push.Run) + } + + pr := job.step(t, "Open or update the sync PR") + if pr.Uses != "" { + t.Fatalf("PR step uses %q, want plain gh (no action; the token split rules out peter-evans/create-pull-request, which takes one token)", pr.Uses) + } + if pr.Env["GH_TOKEN"] != "${{ steps.pr-token.outputs.token }}" { + t.Errorf("PR step env GH_TOKEN = %q, want the minted pr-token", pr.Env["GH_TOKEN"]) + } + if _, leaked := pr.Env["PUSH_TOKEN"]; leaked { + t.Errorf("PR step sets PUSH_TOKEN; the PR step must never see push-token") + } + if pr.Env["BRANCH"] != workerEnvAutosyncBranch { + t.Errorf("PR step env BRANCH = %q, want the fixed branch %q (so a later run updates the same PR instead of opening a new one)", pr.Env["BRANCH"], workerEnvAutosyncBranch) + } + if !strings.Contains(pr.Run, "gh pr create") || !strings.Contains(pr.Run, "gh pr edit") || !strings.Contains(pr.Run, `--add-label "$LABEL"`) { + t.Errorf("PR step run %q, want it to create-or-update the PR and add the label via gh", pr.Run) + } + if pr.Env["LABEL"] != "status/needs-review-auto" { + t.Errorf("PR step env LABEL = %q, want status/needs-review-auto (routes it through the automated review workflow)", pr.Env["LABEL"]) + } + + merge := job.step(t, "Enable auto-merge") + if merge.If != mismatchIf+" && steps.pr.outputs.number" { + t.Errorf("merge step if = %q, want gated on a real mismatch and a PR having been opened", merge.If) + } + if merge.Uses != "" || !strings.Contains(merge.Run, "gh pr merge --auto") { + t.Errorf("merge step = %+v, want a gh pr merge --auto call (arms auto-merge; never merges outright)", merge) + } + if merge.Env["GH_TOKEN"] != "${{ steps.pr-token.outputs.token }}" { + t.Errorf("merge step env GH_TOKEN = %q, want pr-token (enabling auto-merge is PR-shaped, not a push)", merge.Env["GH_TOKEN"]) + } +} + +// TestWorkerEnvAutosyncWorkflowNeverInterpolatesExpressionsInRun: every +// `${{ }}` GitHub would otherwise substitute directly into a shell script +// is instead assigned to a step's `env:` first, so a value that happens to +// contain shell metacharacters (a PR title, a run URL) is data the shell +// sees through a variable, never text GitHub splices into the script +// before bash ever runs it. +func TestWorkerEnvAutosyncWorkflowNeverInterpolatesExpressionsInRun(t *testing.T) { + job := readCIWorkflow(t, workerEnvAutosyncWorkflow).job(t, workerEnvAutosyncJob) + for _, step := range job.Steps { + if strings.Contains(step.Run, "${{") { + t.Errorf("step %q run contains a ${{ }} expression; move it to env: and reference the variable instead:\n%s", step.Name, step.Run) + } + } +} + +// TestWorkerEnvSyncCopyAgainstFixture runs the real tools/rbe/worker-env- +// sync-copy against a scratch checkout and a local gascity (reusing +// rbe_worker_env_test.go's fixture): it overwrites the manifest and the pin +// byte for byte, leaves everything else untouched, and fails loudly rather +// than writing anything when gascity's platforms/BUILD.bazel carries no +// pin to copy. +func TestWorkerEnvSyncCopyAgainstFixture(t *testing.T) { + f := newRBEWorkerEnvFixture(t) + stalePin := rbeTestPin(rbeTestStaleManifest) + beads := f.beads(t, rbeTestStaleManifest, stalePin) + rbeTestWrite(t, filepath.Join(beads, workerEnvSyncCopy), readPolicyFile(t, bazelPolicyRoot(t), workerEnvSyncCopy), 0o755) + // A file the copy must never touch. + rbeTestWrite(t, filepath.Join(beads, "tools/rbe/untouched.txt"), "keep\n", 0o644) + + code, out := f.run(t, beads, workerEnvSyncCopy+" main\n") + if code != 0 { + t.Fatalf("worker-env-sync-copy: exit %d\n%s", code, out) + } + if !strings.Contains(out, "copied gascity main's worker-env ("+f.pin+")") { + t.Errorf("worker-env-sync-copy output %q, want it to report the copied pin %s", out, f.pin) + } + gotManifest, err := os.ReadFile(filepath.Join(beads, rbeWorkerEnvManifest)) + if err != nil || string(gotManifest) != f.manifest { + t.Errorf("worker-env.txt after copy = %q, %v; want gascity's manifest %q", gotManifest, err, f.manifest) + } + gotBuild, err := os.ReadFile(filepath.Join(beads, rbeWorkerPlatformBuild)) + if err != nil || !strings.Contains(string(gotBuild), "\"worker-env\": \""+f.pin+"\",") { + t.Errorf("platforms/BUILD.bazel after copy = %q, %v; want the pin %s", gotBuild, err, f.pin) + } + untouched, err := os.ReadFile(filepath.Join(beads, "tools/rbe/untouched.txt")) + if err != nil || string(untouched) != "keep\n" { + t.Errorf("worker-env-sync-copy touched an unrelated file: %q, %v", untouched, err) + } + + // gascity's platforms/BUILD.bazel with no pin at all: nothing copied. + noPinGascity := t.TempDir() + rbeTestWrite(t, filepath.Join(noPinGascity, "main/tools/rbe/worker-env.txt"), "arch x86_64\n", 0o644) + rbeTestWrite(t, filepath.Join(noPinGascity, "main/platforms/BUILD.bazel"), "platform(\n name = \"rbe_worker\",\n)\n", 0o644) + beadsNoPin := f.beads(t, rbeTestStaleManifest, stalePin) + rbeTestWrite(t, filepath.Join(beadsNoPin, workerEnvSyncCopy), readPolicyFile(t, bazelPolicyRoot(t), workerEnvSyncCopy), 0o755) + cmd := "WORKER_ENV_SYNC_URL=file://" + noPinGascity + " " + workerEnvSyncCopy + " main\n" + code, out = f.run(t, beadsNoPin, cmd) + if code == 0 || strings.Contains(out, "copied gascity") { + t.Errorf("worker-env-sync-copy with no gascity pin: exit %d\n%s, want a failure and nothing copied", code, out) + } + stillStale, err := os.ReadFile(filepath.Join(beadsNoPin, rbeWorkerEnvManifest)) + if err != nil || string(stillStale) != rbeTestStaleManifest { + t.Errorf("worker-env-sync-copy wrote despite no gascity pin: %q, %v", stillStale, err) + } +} diff --git a/tools/rbe/worker-env-sync-copy b/tools/rbe/worker-env-sync-copy new file mode 100755 index 000000000..4f0b30be2 --- /dev/null +++ b/tools/rbe/worker-env-sync-copy @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# worker-env-sync-copy: copy gascity's worker-env manifest and pin here. +# +# tools/rbe/worker-env-sync-copy [REF] +# +# tools/rbe/worker-env-sync only detects an out-of-step pin (exit 1) or that +# it could not compare (exit 2); it never writes. This is the write half, +# run only after that exit-1 detection: it fetches gascity's +# tools/rbe/worker-env.txt and platforms/BUILD.bazel at REF (default main) +# and overwrites beads' copies, byte for byte, the same edit #7419 made by +# hand. It does not touch git (no add, commit, or push) -- the caller +# (worker-env-autosync.yml) stages and opens the PR. +# +# Env: WORKER_ENV_SYNC_URL, gascity's raw-content base (default +# https://raw.githubusercontent.com/gastownhall/gascity). Tests only. +# Run from the repository root. +set -Eeuo pipefail + +ref=${1:-main} +base=${WORKER_ENV_SYNC_URL:-https://raw.githubusercontent.com/gastownhall/gascity}/$ref +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +curl -fsSL --connect-timeout 5 --max-time 15 --retry 2 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt" +curl -fsSL --connect-timeout 5 --max-time 15 --retry 2 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel" + +pin=$(sed -n 's/^ *"worker-env": "\(sha256:[0-9a-f]\{64\}\)",$/\1/p' "$tmp/BUILD.bazel") +if [ -z "$pin" ]; then + echo "worker-env-sync-copy: gascity $ref's platforms/BUILD.bazel has no worker-env pin; nothing copied" >&2 + exit 1 +fi + +cp "$tmp/worker-env.txt" tools/rbe/worker-env.txt +sed -i -E "s#\"worker-env\": \"sha256:[0-9a-f]{64}\",#\"worker-env\": \"$pin\",#" platforms/BUILD.bazel + +ours=$(sed -n 's/^ *"worker-env": "\(sha256:[0-9a-f]\{64\}\)",$/\1/p' platforms/BUILD.bazel) +if [ "$ours" != "$pin" ]; then + echo "worker-env-sync-copy: platforms/BUILD.bazel still pins $ours after the edit, want $pin" >&2 + exit 1 +fi +echo "worker-env-sync-copy: copied gascity $ref's worker-env ($pin)" From 8dbbe78e9a6ff04ee2d6402e206a59365ae526e9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 07:19:18 +0000 Subject: [PATCH 034/151] feat(backend/http): public HTTP backend door, OSS registration, and bd connect (S6) (#7288) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(backend/http): add the public HTTP backend door, cmd/bd registration, and bd connect internal/httpclient (S1-S5) had no production caller for the per-open backends.OpenOptions.Credential seam and no way for a workspace to actually select it. This slice (S6) closes both gaps in OSS's own idiom, without carrying over anything from bd-enterprise's gateway/box-login/storage-profile machinery: - backend/http (bdhttp) is the public, out-of-tree extension door: Register adds "http" to the backend registry and installs its transport, Open/ Handshake dial a server directly with no workspace on disk, and the registered OpenWith hook threads a per-call credential through beads.OpenBestAvailableWith for a multi-tenant embedder. - internal/httpclient/open_with.go is the backends.Backend.OpenWith hook itself: a fresh dial per open (opts.Credential/HTTPClient/UserAgent), distinct from the process-wide registered dialer Open/OpenReadOnly use. - cmd/bd/http_backend.go registers the backend for the OSS binary, so a workspace whose metadata.json names "http" opens through the ordinary OpenBestAvailable path; any other unregistered backend name still refuses. - cmd/bd/connect.go adds the minimal `bd connect ` command: a handshake that must succeed before anything is written, a per-user activation sidecar (.beads/http_target.json, never git-tracked), a plaintext-to- non-loopback refusal without --allow-plaintext, and a backend-switch refusal without --force. No credential is ever accepted on the command line or written to disk. - cmd/bd/doctor/gitignore.go requires the new http_target.json / http_local_metadata.json sidecars be git-ignored, the same way the proxied-server client info already is. Co-Authored-By: Claude Sonnet 5 * fix(storage): keep external-dependency blocking over http unless the server enforces it bd ready / bd close / claim hard-failed against the new http backend with "GetAllDependencyRecords not supported by the http backend": cmd/bd's storage chain wraps every opened store in the externaldeps policy decorator, which needs either the narrow ExternalDependencyQueryStore interface or the broad GetAllDependencyRecords fallback to resolve blocking state, and the http client implemented neither. The decorator now stays on for a remote store. It is skipped only when the server's handshake advertises policy.external_dependencies (the server enforces the policy before answering). Without that capability the client resolves blocking itself from the listing and chunked dependency reads, so an issue with an unresolvable external: dependency stays out of bd ready exactly as it does on a local backend; it is never silently skipped (DESIGN 3.6). Server-side enforcement is slice S10. storage.RemoteBackendStore marks a DoltStorage that is a pure network client of a remote bd serve; the http backend's Store satisfies it. Co-Authored-By: Claude Sonnet 5 * fix(cmd/bd): suppress the CreatedBy stamp on create against a remote backend bd create failed against the http backend with "Issue.CreatedBy: ... is not expressible on the v0 wire": the http client's create role deliberately refuses any POPULATED CreatedBy/CreatedAt (the creation stamp belongs to the server's own journal entry, not a value the caller supplies), but cmd/bd/create.go unconditionally stamped getActorWithGit() onto every created issue regardless of backend. The refusal only fires on a non-zero field, so suppressing the stamp for a registered remote backend is wire- compatible by construction, not a workaround. Add isRemoteBackendWorkspace()/creationActorStamp() to backend_support.go, reusing the existing resolveCommandBeadsDir(dbPath) + backendNameForErrorFraming(beadsDir) + backends.IsRemote(name) pattern already used elsewhere in that file, and use creationActorStamp() at both create.go call sites (the dry-run preview and the real create). Co-Authored-By: Claude Sonnet 5 * test(backend/http): add an embedder linkage test over public packages only Prove the one promise S6 makes to an out-of-tree embedder (gc is the motivating case): importing nothing but github.com/steveyegge/beads, .../backend, and .../backend/http is enough to register the http backend, open a workspace through it with a per-call credential (beads.OpenBestAvailableWith), and read through the role-accessor surface (IssueReader().Ready) against a stand-in server — with no internal/... import in the test file, which is itself the thing under test. Written fresh for OSS beads S6; no bd-enterprise source copied. Co-Authored-By: Claude Sonnet 5 * docs: document the HTTP backend, bd connect, and the public backend/http door Add a README section with bd connect usage, the credential ladder, and the plaintext/force refusals, plus a CHANGELOG entry for the new cmd/bd registration, bd connect command, and the public backend/http package. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): address bee-ghosttrack review on #7288 (connect --clear restore, persisted --allow-plaintext) Closes the CHANGES_REQUESTED items on the HTTP backend door PR: - Must-fix: `bd connect --force` over a workspace with its own local (dolt) identity now has writes succeed instead of tripping "workspace identity mismatch detected" — validateWorkspaceIdentity (cmd/bd/main.go) and the sidecar (internal/httpclient/metadata.go) now compare against the http backend's own pinned project id. New e2e regression test TestE2E_ForceConnectWritesSucceedDespiteDifferingLocalProjectID (backend/http/e2e_cli_test.go), confirmed to fail without the fix. - Should-fix 1: `bd connect --clear` now restores metadata.json's backend selection to whatever it was before this workspace connected, instead of leaving it pinned to "http" with no sidecar to dial. Target.PreviousBackend (internal/httpclient/target.go) is recorded by Attach (backend/http/attach.go) and round-tripped through the sidecar; connect.go reads it back on --clear. New tests in backend/http/attach_test.go and internal/httpclient/target_test.go. - Should-fix 2: `bd connect --allow-plaintext` is now remembered in the sidecar (Target.AllowInsecureCredential), scoped to the target it was granted for, so every later ordinary command's dial honors the grant without also needing BEADS_HTTP_ALLOW_INSECURE=1 set (internal/httpclient/dial.go). Fixed BearerProvider.warnInsecure's wording (internal/httpclient/credential.go), which used to assert a token was being sent immediately above a refusal saying the opposite. New test TestDialWithWiresTargetAllowInsecureCredentialThroughToTheGuard. - Should-fix 3: README no longer claims HTTP mode works "exactly as with a local database" — it lists what round-trips and points at the divergence ledger (engdocs/design/http-divergence-ledger.md) for what differs. - Verified, no code change needed: Attach already handles an existing metadata.json project_id coherently (only cfg.Backend is touched); bd show's isNotFoundErr already matches a http 404 (issueops.ErrNotFound is storage.ErrNotFound, the same sentinel a local miss uses), so it reads as "not found" rather than an unsupported-operation error. - Doc fix: guardInsecureCredential's doc comment and refusal message were stale against should-fix 2's own change — they named only DialOptions.AllowInsecureCredential and the env var as ways to get past the guard, not the newly-persisted target.AllowInsecureCredential or `bd connect --allow-plaintext` itself. Gates run clean on this branch: go build/vet, go test over internal/httpclient, backend/http (including the full e2e suite under BEADS_TEST_EMBEDDED_DOLT=1) and cmd/bd, make api-check, make ci-pr-lint, make bazel-sync-check, bazel run //:gazelle -- -mode=diff, and internal/storage's TestEveryLegWiresEveryRoleContract. Co-Authored-By: Claude Sonnet 5 * test(backend/http): give the e2e server the BatchGetter role #7248 added to the required set Co-Authored-By: Claude Opus 5.5 (1M context) * fix: exact http ready totals, connect capability row, doc corrections Ready totals over http (review major). externaldeps drops externally blocked rows client-side for a store that cannot express ExcludeIDs (today httpclient.Store). dropBlockedIssuesWithCounts compacted the fetched rows in place (`kept := rows[:0]`), and readyWorkWithCountsAndTotal then re-scanned `rows` to count the blocked rows it had dropped. By then the compaction had overwritten them, so whenever a blocked row came back ahead of a kept one the re-scan undercounted the drops: `bd ready --json`'s total and text-mode `bd ready`'s count (CountReadyWork) overstated ready work against any server that does not advertise policy.external_dependencies. Both drop helpers now copy into a fresh slice, and dropBlockedIssuesWithCounts reports how many rows it dropped, so nothing re-scans `rows`. fakeStore gains an excludeIDsUnsupported switch that, like httpclient.Store, refuses any ExcludeIDs. The new TestReadyTotalsWhenStoreCannotExpressExcludeIDs puts the blocked issue first and checks the page and total for an unlimited filter, a Limit whose bumped window holds the whole set, and one the set outgrows, then CountReadyWork. TestServedReadyExcludesUnsatisfiedExternalDependencyWithoutServerCapability now also checks the total and count against a real in-process bd serve. Against the previous decorator the unit test's unlimited case reports total and count 3 beside a 2-row page, and the served test 2 beside a 1-row page. Review minors, all wording: - `bd connect --clear` pointed at `bd config set backend dolt`, which writes the database's config table rather than metadata.json and, in the state that prints the advice (metadata.json says http, sidecar gone), fails the pre-open identity check. It now says to reconnect, or to set "backend" in metadata.json. - CHANGELOG described the superseded skip-at-wrap design; it now describes the always-wrap decorator with its capability probe and client-side drop. - RemoteBackendStore's docs no longer name a consumer; nothing in-tree checks it. - creationActorStamp's comment pointed at backend/http/lifecycle.go; the refusal lives in internal/httpclient/lifecycle.go. Capability row. TestProxyCapabilityRegistryCoversCommandTree fails at 60b7128204: the command tree walk finds `connect`, which this PR adds, with no row in cmd/bd/capability_registry.go ("1 command(s) have no proxied-server capability row: connect"). The test predates the PR (#6582 is an ancestor of its base) and passes at the base dabf20ef07. connect opts out of store init (skipStoreAnnotation) and never reaches the store factory, so it belongs in proxyPermittedPaths beside init and setup. The row is runtime-neutral: validateProxyRegistryBeforeProvider returns nil both for a permitted row and for a path with no row, so this only records the policy decision the coverage test asks for. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * docs(httpclient): regenerate the divergence ledger doc after the S5 rebase The rebase onto feat/http-client-batch-apply's rebased head (S3/S4 merged to main) left the ledger doc's summary line at a conflict-resolution placeholder count. Regenerate it from internal/httpclient/encode.Ledger() so TestDivergenceLedgerDocMatchesLedger passes against the merged ledger.go. Co-Authored-By: Claude Sonnet 5 * fix(backend/http): host-scope S6's test BEADS_HTTP_TOKEN values Rebasing onto feat/http-client-batch-apply's rebased head pulled in main's #7247 review-fix round, which made BEADS_HTTP_TOKEN/BEADS_HTTP_TOKEN_COMMAND host-scoped (host[:port]=value, scopedEnvValue/parseScopedEnv in internal/httpclient/credential.go) after S6's own tests were written against the unscoped form. Five S6 tests across backend/http/open_test.go, backend/http/register_test.go and internal/httpclient/insecure_credential_guard_test.go set a bare token and now hit the new ladder's "an unscoped token would be used for whatever server..." refusal. Scope each to the test server's actual host (127.0.0.1 for the httptest.Server-backed recording servers, nonLoopbackTestHost for the insecure-credential-guard integration tests) so they exercise the same ladder rung as before. Co-Authored-By: Claude Sonnet 5 * build(bazel): resync go_srcs/repo_files for the new backend/http and test/embedder packages make bazel-sync-check failed after the rebase: tools/bazel/go_srcs.py's managed repo_files filegroups for backend/http and test/embedder (new in this slice) hadn't been generated, and the root BUILD.bazel / tools/bazel's release_cross_build target list hadn't picked up the new package either. Regenerate with `python3 tools/bazel/go_srcs.py`; `bazel run //:gazelle -- -mode=diff` reports no further drift. Co-Authored-By: Claude Sonnet 5 * fix(cmd/bd): route swarm's create paths through creationActorStamp after main's currentActor change Co-Authored-By: Claude Opus 5.5 (1M context) * fix: police http's served roles, refuse close --claim-next, exact totals External-dependency policy over http (review blocker). cmd/bd wraps every store, http included, in the externaldeps decorator, and its role accessors rebuilt each role over legacy methods httpclient.Store refuses (SearchIssuesWithCounts, GetDependencyTree, ClaimReadyIssue, IsBlocked, GetDependencyRecordsForIssues, BatchCloserWithPolicy). Over http `bd list`, `bd dep tree`, `bd ready --claim` and a policed `bd close` therefore exited 1, even in a workspace with no external dependency. Over a store that reports storage.RemoteBackendStore, each accessor now starts from the inner store's served role and layers the policy around it (remote_roles.go): - IssueReader serves List and Get from the served reader; Ready and a ready-flagged List go through the policy reader. - IssueClaimer refuses an externally blocked claim before the served one. - ReadyClaimer uses the served claim-next while no issue, closed ones included, holds an unsatisfied external ref. Otherwise it claims down the policy's filtered ready page through the served Claimer, because claimNext cannot exclude ids. - BlockingAnnotator and TreeWalker merge external blockers into, or add external leaves to, the served answer. They read only the named issues' edges through the served EdgeReader, 100 anchors per call. - BatchCloser refuses externally blocked open items and sends the rest to the served closer as one request. --force keeps the base path, and a next claim is refused whole (below). RemoteBackendStore now only chooses this composition. Whether the policy runs is still ServerEnforcesExternalDependencyPolicy's answer (design 3.6), and the storage and httpclient docs say so. remote_roles_test.go drives each role over a store that refuses the legacy methods. TestServedRolesKeepTheExternalDependencyPolicy runs every role over a real http server. TestE2E_BDOverHTTP adds `bd list` (JSON, text and --deps), `bd dep tree`, `bd ready`, `bd list --ready`, `bd ready --claim`, a refused `bd close --claim-next`, and a refused then forced close of an externally blocked issue. At the fixup before the composition, the role tests then written (bar one pin of the next-claim refusal, since replaced) and the e2e's list, dep tree, claim and close steps of that round were red. Close with a next claim over http (review major). batchCloseIssues has no member for a next claim, so httpclient refuses any ClaimNext before the dial (ledger W-CloseBatchRequest.ClaimNext). Over http `bd close --claim-next` therefore always exited 1 and closed nothing. With an external ref anywhere in the workspace (a closed holder or --force included) the decorator's base path failed first, with an opaque `BatchCloserWithPolicy` unsupported. remoteBatchCloser now refuses a next claim whole, before it reads or closes anything, and the error says nothing closed: close without --claim-next, then run `bd ready --claim`. A request that is invalid on every backend is still ErrValidation first, and a server that enforces the policy itself gets the request unchanged. Composing close-then-claim was not taken. The role closes and claims in one transaction and answers with an error or with outcomes, never both, so a claim that failed after the closes committed could not report them; httpclient refuses the same request for that reason. A CLI-level close then claim that can report a partial result is the follow-up. The review also suggested leaving closed holders out of the blocking state. They stay in, as on every backend: a closed issue is never a claim candidate, so they change no claim, and leaving them out would cost a status read per holder because the wire has no batch read. TestRemoteBatchCloserRefusesANextClaim covers an open holder, a closed one, none, --force, and a server that enforces the policy; with the previous remote_roles.go the first four are red. TestRemoteBatchCloserValidatesANextClaimFirst keeps ErrValidation ahead of the refusal. Dropping the filter check, or loading the blocking state before refusing, is each caught. TestE2E_BDOverHTTP runs `bd close --claim-next` while the holder is open and again after it is closed; with the previous remote_roles.go both steps are red and print `operation "BatchCloserWithPolicy" not supported`. Ledger F-close no longer says --claim-next is served, and no longer names a renderEscapedHTTPRefusal that does not exist. L14 says the decorator's claim leg applies while any issue, closed or not, holds an unsatisfied ref. The README and CHANGELOG mention the refusal, and the decorator's IsBlocked, IsBlockedBatch and CloseIssueChecked say they refuse over a store that serves its roles. Ready totals with external blockers (review majors, both rounds). CountReadyWork asks for Limit=0, and readyWorkWithCountsAndTotal took any Limit<=0 to mean "the window held the whole ready set". Over http a zero limit never reaches the wire: the server answers with its default page (workapi.DefaultReadyLimit, 100 rows) while the bridge reports the full total, so past 100 ready issues text-mode `bd ready` kept counting externally blocked issues beyond the first page. Exactness is now judged from the rows that came back (total <= len(rows)). A window that holds only part of the set no longer subtracts every externally blocked id in the workspace either: that set includes closed issues and ones the filter excludes, none of them in the total, so the count came out low. It refetches the filtered ready set, ids only, and subtracts the blocked rows in it; the old bound is left as the fallback for a refetch that comes back truncated too. fakeStore gains the bridge's shape: a defaultPage for a zero Limit, with GetReadyWorkWithCountsAndTotal still totalling the whole set. TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage (a blocked issue past the first page) and TestReadyTotalsIgnoreExternallyBlockedIssuesOutsideTheReadySet (a blocked issue outside the ready set) have served counterparts, TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage and TestServedReadyTotalIgnoresAnExternallyBlockedIssueOutsideTheFilter (an assignee filter). The first pair is red on the Limit<=0 condition, the second on the subtract-every-blocked-id branch. Docs: ledger L8 now says which decorator paths take the chunked GetAllDependencyRecords scan and which read only named edges, and L14 records the decorator's composed claim leg; http-divergence-ledger.md is regenerated. The README names the verified commands, and the README and CHANGELOG say how cross-repo dependencies behave over http. Minors from the reviews: - Drop creationActorStamp/isRemoteBackendWorkspace and restore the base stamps at all 11 sites. The arm guarded a refusal the http create role does not make: since #7247 (actorStampedCreateMember) it accepts a CreatedBy naming the actor. Because the arm keyed on the CWD workspace, a create routed from an http workspace into a local one (--repo) stored an empty created_by. TestE2E_BDOverHTTP checks created_by for a direct and a routed create; the routed check is red with the arm restored. The CHANGELOG sentence describing the arm goes with it. - bdhttp.Attach decides a reconnect from the raw metadata.json backend. GetBackend() reads an unregistered "http" as the dolt default, so in an embedder that never called Register a reconnect recorded dolt as the backend --clear restores. The reconnect test now seeds postgres (TestAttachOnAReconnectKeepsTheEarlierPreviousBackend) and is red with the old check. - README, CHANGELOG and `bd connect --help` document the host-scoped BEADS_HTTP_TOKEN=host[:port]= form (and _COMMAND), and say why a bare value is refused. - bdhttp.Attach also gitignores http_local_metadata.json, the per-user file the store writes on first use, so an embedder activating through Attach alone gets the coverage `bd connect` already had. TestAttachGitignoresBothPerUserFiles pins it. - ServerSnapshot.WireRevision's doc names the handshake's actual check instead of a comparison `bd connect` never makes. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * test(backend/http): give the e2e server the LeaseReclaimer role #7412 added to the required set main's #7412 made LeaseReclaimer part of httpapi's required role set and updated main's own binders, but backend/http's e2eServeConfig predates it, so every TestE2E_* that binds the in-process server failed with "httpapi: no database source". Same shape as e63f7da36a for BatchGetter. Co-Authored-By: Claude Opus 5.5 * fix(externaldeps): lead the --claim-next refusal with the wire fact; pin the L14 leg and connect --clear Review i4 minors/nits on the stranded fixup, carried as a separate commit: - remoteBatchCloser's --claim-next refusal now says the served batch close cannot carry a next claim (the real cause), rather than blaming externally blocked work a workspace may not have. The test asserts the reason, that nothing closed, and the remedy. - GetDependencyTree's doc says a served store refuses its reads and that bd dep tree goes through remoteTreeWalker. - README no longer gives `bd ready --claim` local semantics everywhere; it names when ledger row L14's two-step claim applies. - TestE2E_BDOverHTTP claims once more after the external-ref holder is closed, pinning L14's composed leg (a mutant whose composed leg claims nothing turns only that step red). - New TestE2E_ConnectClearRestoresThePreviousBackend: --clear restores the prior dolt backend with its identity intact, removes the sidecar, and a second --clear changes nothing. Co-Authored-By: Claude Opus 5.5 * test(backend/http): run the served e2e under bazel //backend/http:http_test declared no bd binary, so under bazel the e2e files (bazeltest.PrebuiltBD) failed on BEADS_TEST_BD_BINARY with BEADS_HTTP_TEST_REQUIRED=1 and only "passed" by skipping otherwise. It now takes //cmd/bd:bd_for_tests (visibility added), size large / timeout long: the served cases run 4-5 minutes on the farm under BEADS_TEST_EMBEDDED_DOLT=1. Co-Authored-By: Claude Opus 5.5 * test(backend/http): run the e2e tier in the embedded lane as http_served_test The previous commit gives //backend/http:http_test the bd binary, but no lane sets BEADS_TEST_EMBEDDED_DOLT for it, so its TestE2E_* cases still skipped in CI. http_served_test runs the same test binary through //tools/bazel:go_test_variant.sh with BEADS_TEST_EMBEDDED_DOLT=1 and BEADS_HTTP_TEST_REQUIRED=1 (a skip fails it). It is tagged embedded, so the --config=embedded lane runs it and prcore/ci keep their current shape, the same split as //internal/httpclient:httpclient_served_test. Under plain --config=embedded all 8 TestE2E_* cases run, none skipped. TestBazelRetiredLanesCannotBeNarrowed pins every embedded-lane target's args and env; the new target is added to that pin list. Co-Authored-By: Claude Opus 5.5 * fix(externaldeps): cap the ready rows kept, not the bumped http window An http store cannot carry ExcludeIDs on listReadyWork, so readyExclusion fetches the unfiltered ready set, widened by len(blocked) under a limit, and drops the externally blocked rows client-side. The http ready bridge enforced MaxRows on the rows it fetched rather than the rows kept. So `bd ready --limit N --max-rows N` exited 2 whenever more than N issues were ready and any workspace issue held an unsatisfied external blocker (a closed one counts), even with no blocked row in the window, and a blocked row in the page counted against the cap although the caller never sees it. Locally neither can happen: issueops enforces the cap on the filtered page it delivers, after the Limit trim (finishReadyWorkWithCounts). The client-side path now takes MaxRows off the inner query and enforces the caller's cap on the rows it keeps, after the trim, in all three listings. The inner query keeps the cap whenever nothing is dropped client-side. MaxRows never rode the wire, so the fetch is the size it was; refetchedReadyTotal already clears the cap on its own over-fetch. TestServedReadyMaxRowsCountsTheRowsKept runs the decorator over a served http store holding three free issues and one externally blocked one: a cap equal to the limit, a cap under a wider limit, an unlimited listing, and a kept page wider than the cap, which must still return ErrTooManyRows with the caller's cap and source. Each case runs through GetReadyWork, GetReadyWorkWithCounts and GetReadyWorkWithCountsAndTotal. Leaving the cap on the inner query fails the first three cases in all three methods, dropping the re-applied cap fails the fourth, and enforcing it before the Limit trim fails the first. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Eddie the Engineer --- BUILD.bazel | 8 + CHANGELOG.md | 41 + README.md | 70 ++ backend/http/BUILD.bazel | 149 +++ backend/http/attach.go | 133 +++ backend/http/attach_test.go | 180 +++ backend/http/credential.go | 57 + backend/http/e2e_cli_test.go | 1059 +++++++++++++++++ backend/http/errors.go | 111 ++ backend/http/http.go | 216 ++++ backend/http/open_test.go | 235 ++++ backend/http/register_test.go | 318 +++++ backend/http/target.go | 41 + cmd/bd/BUILD.bazel | 5 + cmd/bd/capability_registry.go | 2 +- cmd/bd/connect.go | 324 +++++ cmd/bd/context_cmd.go | 15 +- cmd/bd/doctor/gitignore.go | 7 +- cmd/bd/http_backend.go | 16 + cmd/bd/main.go | 23 + cmd/bd/storage_chain.go | 11 + engdocs/design/http-divergence-ledger.md | 8 +- internal/gitignore/BUILD.bazel | 10 +- internal/gitignore/ensure.go | 42 + internal/gitignore/ensure_test.go | 64 + internal/httpclient/BUILD.bazel | 6 + internal/httpclient/completeness_test.go | 1 - internal/httpclient/credential.go | 19 +- internal/httpclient/dial.go | 22 + internal/httpclient/dial_test.go | 5 + internal/httpclient/encode/ledger.go | 19 +- .../httpclient/insecure_credential_guard.go | 116 ++ .../insecure_credential_guard_test.go | 251 ++++ internal/httpclient/metadata.go | 5 +- internal/httpclient/open_with.go | 69 ++ internal/httpclient/open_with_test.go | 67 ++ internal/httpclient/roles_graph.go | 100 ++ .../served_external_dependency_policy_test.go | 492 ++++++++ internal/httpclient/store.go | 54 + internal/httpclient/target.go | 77 +- internal/httpclient/target_test.go | 61 + internal/httpclient/unsupported_gen.go | 7 +- internal/httpclient/wire/handshake.go | 22 +- internal/storage/externaldeps/BUILD.bazel | 2 + internal/storage/externaldeps/batch_closer.go | 5 + internal/storage/externaldeps/decorator.go | 260 +++- .../storage/externaldeps/decorator_test.go | 292 ++++- internal/storage/externaldeps/remote_roles.go | 412 +++++++ .../storage/externaldeps/remote_roles_test.go | 636 ++++++++++ .../storage/externaldeps/role_accessors.go | 29 +- internal/storage/storage.go | 49 + scripts/pr_risk_bazel_coverage_test.go | 4 + test/embedder/BUILD.bazel | 73 ++ test/embedder/embedder_test.go | 253 ++++ tools/bazel/BUILD.bazel | 1 + 55 files changed, 6472 insertions(+), 82 deletions(-) create mode 100644 backend/http/BUILD.bazel create mode 100644 backend/http/attach.go create mode 100644 backend/http/attach_test.go create mode 100644 backend/http/credential.go create mode 100644 backend/http/e2e_cli_test.go create mode 100644 backend/http/errors.go create mode 100644 backend/http/http.go create mode 100644 backend/http/open_test.go create mode 100644 backend/http/register_test.go create mode 100644 backend/http/target.go create mode 100644 cmd/bd/connect.go create mode 100644 cmd/bd/http_backend.go create mode 100644 internal/gitignore/ensure.go create mode 100644 internal/gitignore/ensure_test.go create mode 100644 internal/httpclient/insecure_credential_guard.go create mode 100644 internal/httpclient/insecure_credential_guard_test.go create mode 100644 internal/httpclient/open_with.go create mode 100644 internal/httpclient/open_with_test.go create mode 100644 internal/httpclient/served_external_dependency_policy_test.go create mode 100644 internal/storage/externaldeps/remote_roles.go create mode 100644 internal/storage/externaldeps/remote_roles_test.go create mode 100644 test/embedder/BUILD.bazel create mode 100644 test/embedder/embedder_test.go diff --git a/BUILD.bazel b/BUILD.bazel index 134a91d1d..104409817 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -299,6 +299,7 @@ filegroup( ) + [ "//backend:repo_doc_files", "//backend/conformance:repo_doc_files", + "//backend/http:repo_doc_files", "//beadserrors:repo_doc_files", "//cmd/bd:repo_doc_files", "//cmd/bd/doctor:repo_doc_files", @@ -442,6 +443,7 @@ filegroup( "//scripts/workflowtags:repo_doc_files", "//test/conformance:repo_doc_files", "//test/docsync:repo_doc_files", + "//test/embedder:repo_doc_files", "//test/testmainconvention:repo_doc_files", "//tests/hook_timeout_backends:repo_doc_files", "//tests/migration:repo_doc_files", @@ -530,6 +532,7 @@ filegroup( ) + [ "//backend:repo_go_srcs", "//backend/conformance:repo_go_srcs", + "//backend/http:repo_go_srcs", "//beadserrors:repo_go_srcs", "//cmd/bd:repo_go_srcs", "//cmd/bd/doctor:repo_go_srcs", @@ -673,6 +676,7 @@ filegroup( "//scripts/workflowtags:repo_go_srcs", "//test/conformance:repo_go_srcs", "//test/docsync:repo_go_srcs", + "//test/embedder:repo_go_srcs", "//test/testmainconvention:repo_go_srcs", "//tests/hook_timeout_backends:repo_go_srcs", "//tests/migration:repo_go_srcs", @@ -746,6 +750,7 @@ filegroup( ) + [ "//backend:repo_go_test_srcs", "//backend/conformance:repo_go_test_srcs", + "//backend/http:repo_go_test_srcs", "//beadserrors:repo_go_test_srcs", "//cmd/bd:repo_go_test_srcs", "//cmd/bd/doctor:repo_go_test_srcs", @@ -889,6 +894,7 @@ filegroup( "//scripts/workflowtags:repo_go_test_srcs", "//test/conformance:repo_go_test_srcs", "//test/docsync:repo_go_test_srcs", + "//test/embedder:repo_go_test_srcs", "//test/testmainconvention:repo_go_test_srcs", "//tests/hook_timeout_backends:repo_go_test_srcs", "//tests/migration:repo_go_test_srcs", @@ -971,6 +977,7 @@ filegroup( ) + [ "//backend:repo_other_files", "//backend/conformance:repo_other_files", + "//backend/http:repo_other_files", "//beadserrors:repo_other_files", "//cmd/bd:repo_other_files", "//cmd/bd/doctor:repo_other_files", @@ -1114,6 +1121,7 @@ filegroup( "//scripts/workflowtags:repo_other_files", "//test/conformance:repo_other_files", "//test/docsync:repo_other_files", + "//test/embedder:repo_other_files", "//test/testmainconvention:repo_other_files", "//tests/hook_timeout_backends:repo_other_files", "//tests/migration:repo_other_files", diff --git a/CHANGELOG.md b/CHANGELOG.md index 20024c147..7945855c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 will not send, and value constraints (`maxLength`, `pattern` and the like) are outside the digest: such changes need their own review against `wire_revision`. +- `cmd/bd` registers the http client backend built on top of the v0 `bd + serve` wire (`internal/httpclient`), so a workspace whose + `.beads/metadata.json` selects `"backend": "http"` opens over HTTP through + the ordinary `OpenBestAvailable` path, exactly like a registered extension + backend does. A new `bd connect ` command performs the handshake + (`api_version`, wire revision, and — with `--expect-project-id` — workspace + identity) and writes nothing until it succeeds; it then records the + per-user activation sidecar (`.beads/http_target.json`, never + git-tracked — `cmd/bd/doctor/gitignore.go` now requires it and its local + metadata sidecar `http_local_metadata.json` be ignored) and sets + `metadata.json`'s backend to `"http"`. `bd connect` refuses a plain `http://` + URL to a non-loopback host (a bearer credential would cross the network in + the clear) unless `--allow-plaintext` is given, and refuses to switch a + workspace that already selects a different backend unless `--force`. A + credential is never accepted on the command line or written to disk; it + comes from the same ladder every http request already uses + (`BEADS_HTTP_TOKEN=host[:port]=`, then + `BEADS_HTTP_TOKEN_COMMAND=host[:port]=`, then the credentials + file, then none); both variables take only that host-scoped form, and a + bare value is refused, since it would be sent to whatever server the + workspace's `http_target.json` names. The external-dependency policy + decorator (`internal/storage/externaldeps`) still wraps an http store, and + stands down only when the server's handshake advertises + `policy.external_dependencies` (that server has already applied the policy + before answering); against any other server it applies the policy + client-side, around the http store's own served roles. Externally blocked + issues drop out of `bd ready`'s page, total, and count (the v0 wire cannot + express an id exclusion) and out of `bd ready --claim`, `bd close` refuses + them without `--force`, and `bd dep tree` shows their external leaves. + `bd close --claim-next` refuses over http before anything closes, since + the wire's batch close cannot carry the claim; its error says to close + without it and then run `bd ready --claim`. +- The public `backend/http` package (`bdhttp`) is the out-of-tree door onto + this backend for an embedder that links beads as a library rather than + running `cmd/bd`: `Register(Options)` adds `"http"` to the registry and + installs its transport, `Open`/`Handshake` dial a server directly with no + workspace on disk, and the existing public `beads.OpenBestAvailableWith` + takes a per-call `OpenOptions.Credential` through it — the case a + multi-tenant embedder serving many workspaces needs, where a + process-global credential cannot stand in for one tenant's own. + EXPERIMENTAL, pin an exact beads version. ### Changed diff --git a/README.md b/README.md index d54c13949..efe3f223b 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,76 @@ are unaffected. the guard with a warning on stderr. Use this only if you know the forward migrations are additive and safe for your specific workload. +## 🔌 HTTP Backend (experimental) + +A workspace can also talk to a remote `bd serve` over HTTP instead of opening +a local Dolt database — useful for a thin client against a shared server, or +for embedding beads in another program as a library. + +```bash +# Point this workspace at a server (verifies the server first; writes nothing on failure) +bd connect https://bd.example.com --expect-project-id my-project + +# The everyday commands work the same way: create, show, list, update, close, +# dependencies (including `bd dep tree`), comments, ready/count queries, and +# batch apply all round-trip over the wire with the same semantics as a local +# database. So does `bd ready --claim`, except against an older server or while +# any issue holds an unsatisfied external ref: then it lists and claims in two +# steps rather than one transaction (ledger row L14, linked below). +bd create "Fix auth bug" -p 1 -t bug +bd ready --json +bd close bd-a1b2 "Fixed" + +# Loopback needs no TLS +bd connect http://127.0.0.1:8080 + +# Forget the server and restore this workspace's previous backend selection +bd connect --clear +``` + +HTTP mode is not a byte-for-byte mirror of local mode, though: some +operations are knowingly refused (the wire has no way to express them) or +degrade to a different-but-not-misreadable behavior (for example, a few +unbounded listing shapes lose single-query snapshot isolation and instead +walk the server page by page). Every one of these is a deliberate, tested +row, never a silent gap — see +[`engdocs/design/http-divergence-ledger.md`](engdocs/design/http-divergence-ledger.md) +for the full, generated list of what differs and why before depending on an +HTTP-backed workspace for a workflow you haven't checked against it. One +refusal to know up front: `bd close --claim-next` closes nothing over HTTP, +because the wire's batch close cannot carry the claim. Close without it, then +run `bd ready --claim`. + +[Cross-repo dependencies](docs/core-concepts/dependencies.md#cross-repo-dependencies) +keep their policy over HTTP. The client resolves each `external:` reference +against its own `external_projects` configuration. `bd dep tree` shows the +reference as a leaf, and while it is unsatisfied the issue stays out of +`bd ready` and `bd ready --claim`, and `bd close` refuses it without +`--force`. A server that advertises enforcing that policy itself is trusted +to do so instead. + +`bd connect` writes two things on success: `.beads/metadata.json` gets +`"backend": "http"`, and a per-user, never-git-tracked sidecar +(`.beads/http_target.json`) records the server URL and pinned project id. It +never writes a credential to disk. A bearer credential, if the server +requires one, comes from `BEADS_HTTP_TOKEN=host[:port]=`, then +`BEADS_HTTP_TOKEN_COMMAND=host[:port]=` (a helper that prints a +token), then the credentials file, then no credential at all. Both variables +must name the server they are for (a pattern without a port matches that +host on any port); a bare value is refused, because it would be sent to +whatever server the workspace's `http_target.json` names. Connecting to a +non-loopback server over plain `http://` is refused by default (a bearer +credential would cross the network unencrypted) — pass `--allow-plaintext` to +override, or use `https://`. Switching a workspace that already selects a +different backend needs `--force`. + +An embedding program that links beads as a library rather than running +`cmd/bd` registers this backend itself through the public +[`backend/http`](backend/http) package (`bdhttp.Register`), and can open a +workspace with a credential scoped to one caller via +`beads.OpenBestAvailableWith` — see that package's doc comment for the +multi-tenant case. This surface is EXPERIMENTAL: pin an exact beads version. + ## 🌐 Community Tools See [docs/community-tools.md](docs/community-tools.md) for a curated list of community-built UIs, extensions, and integrations—including terminal interfaces, web UIs, editor extensions, and native apps. diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel new file mode 100644 index 000000000..3411699a1 --- /dev/null +++ b/backend/http/BUILD.bazel @@ -0,0 +1,149 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") + +go_library( + name = "http", + srcs = [ + "attach.go", + "credential.go", + "errors.go", + "http.go", + "target.go", + ], + importpath = "github.com/steveyegge/beads/backend/http", + visibility = ["//visibility:public"], + deps = [ + "//backend", + "//internal/configfile", + "//internal/gitignore", + "//internal/httpapi/apigen", + "//internal/httpclient", + "//internal/httpclient/wire", + "//internal/storage/backends", + ], +) + +go_test( + name = "http_test", + # The served e2e cases run ~4-5 min on the farm under + # BEADS_TEST_EMBEDDED_DOLT=1, past the medium 300s action limit. + size = "large", + timeout = "long", + srcs = [ + "attach_test.go", + "e2e_cli_test.go", + "open_test.go", + "register_test.go", + ], + # The e2e files exec the real bd (bazeltest.PrebuiltBD); they run under + # BEADS_TEST_EMBEDDED_DOLT=1 and skip otherwise. + data = ["//cmd/bd:bd_for_tests"], + env = {"BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)"}, + deps = [ + ":http", + "//backend", + "//internal/configfile", + "//internal/httpapi", + "//internal/httpclient", + "//internal/storage", + "//internal/storage/backends", + "//internal/storage/domain", + "//internal/storage/embeddeddolt", + "//internal/testutil/bazeltest", + ], +) + +# http_test again, with the e2e tier on: BEADS_TEST_EMBEDDED_DOLT=1 starts the +# in-process server over embedded Dolt and BEADS_HTTP_TEST_REQUIRED=1 turns a +# skip of that tier into a failure. http_test alone skips the TestE2E_* cases +# in every lane, because no lane sets those variables for it. It repeats +# http_test's `env` (go_test_variant.sh). +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "http_served_test", + size = "large", + # -test.timeout=19m fires (with a goroutine dump) before Bazel's 1200s + # action limit (eternal under --config=embedded). + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :http_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + ], + data = [ + ":http_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_HTTP_TEST_REQUIRED": "1", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_EMBEDDED_DOLT": "1", + }, + tags = ["embedded"], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/backend/http/attach.go b/backend/http/attach.go new file mode 100644 index 000000000..f2175ff5f --- /dev/null +++ b/backend/http/attach.go @@ -0,0 +1,133 @@ +// Written fresh for OSS beads S6 (no bd-enterprise source copied): closes the +// S6 review's MED-8 finding, which asked for one public helper an embedder +// (or `bd connect`) can call to activate a workspace for this backend, +// instead of each caller hand-assembling the two-file write itself. +package bdhttp + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/gitignore" + "github.com/steveyegge/beads/internal/httpclient" +) + +// Attach activates the http backend for the workspace at beadsDir: it writes +// the per-user activation sidecar (SaveTarget) pinning target's server and +// project, then sets the workspace's git-tracked backend selection in +// metadata.json to this backend, preserving every other configured field +// (database, project id, ...) and creating a fresh default metadata.json only +// when none exists yet. Both files are written atomically (temp file plus +// rename, 0600): see SaveTarget and configfile.Config.Save. +// +// Unless the caller already set target.PreviousBackend, Attach fills it in +// from the workspace's PRIOR backend selection, so `bd connect --clear` can +// restore it later; see that field's doc. +// +// Attach does NOT verify target itself — it has no network access and takes +// target on faith. A caller MUST call Handshake (or httpclient.Handshake) +// against target first and only call Attach once that succeeds; attaching an +// unverified target lets a workspace record a server that is wrong, hostile, +// or simply never answers. Connect, below, does both in the right order and +// is the easier door for a caller that does not need them split. +// +// Attach does not create beadsDir itself — MkdirAll is the caller's job, same +// as SaveTarget's own precondition. It DOES ensure both of this backend's +// per-user filenames are covered by beadsDir's .gitignore +// (gitignore.EnsurePatternIgnored): the sidecar, and +// httpclient.LocalMetadataFileName, which the store writes on first use. That +// is the same guarantee `bd connect` makes via doctor.EnsureGitignoreForBeadsDir +// — but through a small, independent implementation here rather than +// importing cmd/bd/doctor, whose package as a whole pulls in dolt and +// git-process dependencies that would break backend/http's promise of a +// minimal dependency footprint for an embedder. The two are complementary and +// idempotent together. +// +// It is the one place that writes BOTH of the files that activate beads/http +// (the sidecar and metadata.json's backend selection; the local metadata file +// is the store's own), so a caller that used to hand-assemble the pair +// (`bd connect`'s own connectCmd.RunE, test/embedder's linkage proof) can call +// this instead and stay correct across a future change to either file's shape. +func Attach(beadsDir string, target Target) error { + cfg, err := configfile.Load(beadsDir) + if err != nil { + return fmt.Errorf("reading %s: %w", configfile.ConfigFileName, err) + } + if cfg == nil { + cfg = configfile.DefaultConfig() + } + // target.PreviousBackend (bee-ghosttrack CHANGES_REQUESTED on #7288, + // should-fix 1) is what `bd connect --clear` restores metadata.json's + // backend selection to. A caller rarely sets it itself (connect.go + // doesn't), so fill it in here unless the workspace is ALREADY on this + // backend — a re-connect (a new URL, or re-pinning the same one) must + // not overwrite an earlier recorded previous backend with "http" itself, + // or a chain of reconnects would forget what to restore and --clear + // would land back on http with no sidecar. Carry the ALREADY-recorded + // value forward instead; a failed or missing read just leaves it "", + // the same as a workspace with nothing to restore. The check reads the + // raw field: in a process that never ran Register, GetBackend() reports an + // http workspace as the dolt default, and the reconnect would record that. + if target.PreviousBackend == "" { + if cfg.Backend == httpclient.Backend { + if prior, err := httpclient.LoadTarget(beadsDir); err == nil { + target.PreviousBackend = prior.PreviousBackend + } + } else { + target.PreviousBackend = cfg.GetBackend() + } + } + if err := SaveTarget(beadsDir, target); err != nil { + return fmt.Errorf("writing %s: %w", httpclient.TargetFileName, err) + } + // Both of this backend's per-user files: the sidecar just written, and + // the local metadata file the store itself writes on first use, which + // never passes through here and so must be covered in advance. + for _, name := range []string{httpclient.TargetFileName, httpclient.LocalMetadataFileName} { + if err := gitignore.EnsurePatternIgnored(beadsDir, name); err != nil { + return fmt.Errorf("ensuring %s is gitignored: %w", name, err) + } + } + // Only the backend SELECTION changes here. cfg.Database/cfg.ProjectID (if + // an existing metadata.json carried them for a different, prior backend) + // are left exactly as loaded — see connect.go's own comment on this same + // hazard: this backend's identity lives entirely in the sidecar just + // written above, and clobbering those fields would corrupt a future + // `bd connect --clear` back to a different backend. + cfg.Backend = httpclient.Backend + if err := cfg.Save(beadsDir); err != nil { + return fmt.Errorf("writing %s: %w", configfile.ConfigFileName, err) + } + return nil +} + +// Connect is Handshake followed by Attach: it verifies target — the same +// server-identity check `bd connect` performs before it ever writes +// anything — and only then activates the workspace. It is the convenience +// door for a caller that wants Attach's ordering guarantee (never attach a +// workspace to a server this process has not handshaken) without composing +// Handshake and Attach itself. +// +// Mirroring `bd connect`'s own project-id pinning: if target.ExpectProjectID +// is empty, Connect pins it to whatever project the server answered with +// before calling Attach, so the workspace is tied to that project from this +// call on. If it is already set, Handshake itself refuses a mismatch and +// Connect returns that error without calling Attach at all. Connect does not +// replicate `bd connect`'s other CLI-level policy — --force re-pinning an +// already-attached workspace, or refusing to switch a workspace from a +// different backend — those stay the command's own job; Connect's contract +// is exactly "handshake, then attach". +func Connect(ctx context.Context, beadsDir string, target Target, opts Options) (*ServerSnapshot, error) { + snapshot, err := Handshake(ctx, target, opts) + if err != nil { + return nil, err + } + if target.ExpectProjectID == "" { + target.ExpectProjectID = snapshot.ProjectID + } + if err := Attach(beadsDir, target); err != nil { + return nil, err + } + return snapshot, nil +} diff --git a/backend/http/attach_test.go b/backend/http/attach_test.go new file mode 100644 index 000000000..7594b3240 --- /dev/null +++ b/backend/http/attach_test.go @@ -0,0 +1,180 @@ +package bdhttp_test + +// Written fresh for OSS beads S6: pins the bee-ghosttrack CHANGES_REQUESTED +// finding on #7288, should-fix 1 — `bd connect --clear` restoring a +// workspace's previous backend selection needs Attach to have recorded what +// that previous backend WAS, onto the sidecar, at connect time. + +import ( + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + bdhttp "github.com/steveyegge/beads/backend/http" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/httpclient" +) + +func mustAttachTestURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("url.Parse(%q): %v", raw, err) + } + return u +} + +// TestAttachRecordsThePriorBackendAsPreviousBackend is the first-connect +// case: a workspace that selected "dolt" before ever running `bd connect` +// must have Attach record "dolt" as PreviousBackend, so --clear can restore +// it later. +func TestAttachRecordsThePriorBackendAsPreviousBackend(t *testing.T) { + beadsDir := t.TempDir() + cfg := &configfile.Config{Backend: "dolt", Database: "beads.db"} + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("seed metadata.json: %v", err) + } + + target := bdhttp.Target{BaseURL: mustAttachTestURL(t, "http://127.0.0.1:8080")} + if err := bdhttp.Attach(beadsDir, target); err != nil { + t.Fatalf("Attach: %v", err) + } + + got, err := bdhttp.LoadTarget(beadsDir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.PreviousBackend != "dolt" { + t.Errorf("PreviousBackend = %q, want %q", got.PreviousBackend, "dolt") + } + + after, err := configfile.Load(beadsDir) + if err != nil { + t.Fatalf("reload metadata.json: %v", err) + } + // Asserts on the raw stored field, not GetBackend(): GetBackend() falls + // back to BackendDolt for any name backendnames.Has doesn't recognize + // (TestGetBackendAllowlist in internal/configfile), and "http" is only + // ever added there by bdhttp.Register/httpclient.Register — a + // once-per-process, panics-on-duplicate production wiring call this test + // package must not make. The raw field is exactly what Attach controls. + if after.Backend != httpclient.Backend { + t.Errorf("metadata.json backend = %q after Attach, want %q", after.Backend, httpclient.Backend) + } +} + +// TestAttachOnAFreshWorkspaceRecordsTheImplicitDoltDefault covers a +// workspace with no metadata.json at all: GetBackend()'s own default +// ("dolt") is what a future --clear restores to, matching what every other +// backend-selection read in this codebase already treats as this +// workspace's backend. +func TestAttachOnAFreshWorkspaceRecordsTheImplicitDoltDefault(t *testing.T) { + beadsDir := t.TempDir() + target := bdhttp.Target{BaseURL: mustAttachTestURL(t, "http://127.0.0.1:8080")} + if err := bdhttp.Attach(beadsDir, target); err != nil { + t.Fatalf("Attach: %v", err) + } + got, err := bdhttp.LoadTarget(beadsDir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.PreviousBackend != "dolt" { + t.Errorf("PreviousBackend = %q, want %q (configfile.BackendDolt)", got.PreviousBackend, "dolt") + } +} + +// TestAttachOnAReconnectKeepsTheEarlierPreviousBackend covers a workspace +// ALREADY on http (a bare re-connect, or --force re-pinning the same +// backend): Attach must not overwrite an already-recorded PreviousBackend +// with "http" itself, or a chain of reconnects would forget what --clear is +// supposed to restore. +func TestAttachOnAReconnectKeepsTheEarlierPreviousBackend(t *testing.T) { + beadsDir := t.TempDir() + // Not dolt: an unregistered "http" reads back through GetBackend() as the + // dolt default, so a dolt prior could not tell carrying it forward from + // recording that default. + cfg := &configfile.Config{Backend: "postgres", Database: "beads.db"} + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("seed metadata.json: %v", err) + } + + first := bdhttp.Target{BaseURL: mustAttachTestURL(t, "http://127.0.0.1:8080")} + if err := bdhttp.Attach(beadsDir, first); err != nil { + t.Fatalf("first Attach: %v", err) + } + + // A second connect to a different url, same workspace, same backend + // ("http", already selected) — the shape `bd connect --force` to a new + // server produces for an already-http workspace. + second := bdhttp.Target{BaseURL: mustAttachTestURL(t, "http://127.0.0.1:9090")} + if err := bdhttp.Attach(beadsDir, second); err != nil { + t.Fatalf("second Attach: %v", err) + } + + got, err := bdhttp.LoadTarget(beadsDir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.PreviousBackend != "postgres" { + t.Errorf("PreviousBackend after a reconnect = %q, want the original %q carried forward, not the just-left \"http\"", got.PreviousBackend, "postgres") + } + if got.BaseURL.String() != "http://127.0.0.1:9090" { + t.Errorf("BaseURL = %q, want the second connect's url", got.BaseURL.String()) + } +} + +// TestAttachHonorsACallerSuppliedPreviousBackend covers Connect's doc +// contract: a caller that already set target.PreviousBackend itself (none +// of this package's own callers do today) must have that value win over +// Attach's own inference. +func TestAttachHonorsACallerSuppliedPreviousBackend(t *testing.T) { + beadsDir := t.TempDir() + cfg := &configfile.Config{Backend: "dolt", Database: "beads.db"} + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("seed metadata.json: %v", err) + } + + target := bdhttp.Target{BaseURL: mustAttachTestURL(t, "http://127.0.0.1:8080"), PreviousBackend: "mysql"} + if err := bdhttp.Attach(beadsDir, target); err != nil { + t.Fatalf("Attach: %v", err) + } + got, err := bdhttp.LoadTarget(beadsDir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.PreviousBackend != "mysql" { + t.Errorf("PreviousBackend = %q, want the caller-supplied %q to win over the inferred %q", got.PreviousBackend, "mysql", "dolt") + } +} + +// TestAttachGitignoresBothPerUserFiles pins Attach's .gitignore guarantee for +// both files this backend keeps per user in beadsDir: the sidecar Attach +// writes, and the local metadata file the store writes later, on first use. +// `bd connect` also covers the second through doctor's required patterns, +// but an embedder activating through Attach alone has only this. A reconnect +// must not duplicate either line. +func TestAttachGitignoresBothPerUserFiles(t *testing.T) { + beadsDir := t.TempDir() + for _, raw := range []string{"http://127.0.0.1:8080", "http://127.0.0.1:9090"} { + if err := bdhttp.Attach(beadsDir, bdhttp.Target{BaseURL: mustAttachTestURL(t, raw)}); err != nil { + t.Fatalf("Attach(%s): %v", raw, err) + } + } + content, err := os.ReadFile(filepath.Join(beadsDir, ".gitignore")) + if err != nil { + t.Fatalf("read .gitignore after Attach: %v", err) + } + for _, name := range []string{httpclient.TargetFileName, httpclient.LocalMetadataFileName} { + count := 0 + for _, line := range strings.Split(string(content), "\n") { + if strings.TrimSpace(line) == name { + count++ + } + } + if count != 1 { + t.Errorf(".gitignore names %s %d time(s) after two Attaches, want exactly 1:\n%s", name, count, content) + } + } +} diff --git a/backend/http/credential.go b/backend/http/credential.go new file mode 100644 index 000000000..5152319f3 --- /dev/null +++ b/backend/http/credential.go @@ -0,0 +1,57 @@ +package bdhttp + +import ( + "net/url" + + "github.com/steveyegge/beads/internal/httpclient" +) + +// CredentialProvider authorizes outbound requests to a bd serve. +// +// An implementation must be safe for concurrent use, must never log or +// persist a secret, and must FAIL CLOSED: a configured source that errors +// aborts the request rather than silently downgrading to an unauthenticated +// one. +type CredentialProvider = httpclient.CredentialProvider + +const ( + // TokenEnv carries the bearer token itself, the highest rung of the + // built-in ladder. + // #nosec G101 -- the NAME of an environment variable, not a credential. + TokenEnv = httpclient.TokenEnv + // TokenCommandEnv names a helper that prints a token, either bare or in + // the kubectl ExecCredential envelope {"token","expirationTimestamp"}. + // #nosec G101 -- the NAME of an environment variable, not a credential. + TokenCommandEnv = httpclient.TokenCommandEnv +) + +// BearerProvider is the built-in credential ladder: TokenEnv, then +// TokenCommandEnv, then the credentials file's [host:port] section, then no +// credential at all (the tip OSS server's loopback-trust posture, a +// legitimate answer and not a failure). +type BearerProvider = httpclient.BearerProvider + +// NewBearerProvider builds the built-in ladder for a server. Open and +// Handshake already dial with it; it is exported so an embedder that wants to +// WRAP the default (decorate it for one target, delegate to it for the rest) +// does not have to reimplement it. +func NewBearerProvider(base *url.URL) *BearerProvider { + return httpclient.NewBearerProvider(base) +} + +// ProvidedCredential adapts a CredentialProvider into the opaque +// backends.Credential marker interface, so it can be carried through +// beads.OpenOptions / beads.OpenBestAvailableWith (and, directly, through +// backends.OpenOptions) to the registered "http" backend's OpenWith hook +// Register installs. +// +// A multi-tenant embedder (for example gc, Gas City: one process serving many +// workspaces, called "cities") that wants a distinct credential per workspace +// wraps that workspace's own provider in a ProvidedCredential rather than +// calling Open directly: the registered dialer Register installs is +// process-wide and cannot hold one credential per tenant, but the per-open +// OpenWith seam can, and this is the opaque wrapper type that seam +// recognizes. See httpclient.ResolveCredential and +// backends.OpenOptions.Credential's own doc comment for the single-tenant +// ambient-ladder default this exists to opt out of. +type ProvidedCredential = httpclient.ProvidedCredential diff --git a/backend/http/e2e_cli_test.go b/backend/http/e2e_cli_test.go new file mode 100644 index 000000000..096926365 --- /dev/null +++ b/backend/http/e2e_cli_test.go @@ -0,0 +1,1059 @@ +//go:build cgo + +// Written fresh for OSS beads S6 (no bd-enterprise source copied). The +// in-process server wiring mirrors internal/httpclient/served_harness_test.go +// (itself under provenance there), reused here as a black-box harness: this +// file never imports the wire client, it shells out to a real `bd` binary so +// the CLI dispatch (cmd/bd/http_backend.go's Register call, `bd connect`, +// `bd update --if-revision`'s exit code contract) is what gets exercised, not +// the internal client package. +package bdhttp_test + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +// e2eProjectID is the identity the in-process server publishes; `bd connect` +// reads it back off the handshake and pins the workspace to it. +const e2eProjectID = "proj-e2e-s6" + +const e2eDatabase = "httpe2e" + +// TestE2E_BDOverHTTP drives the real `bd` binary against an in-process `bd +// serve`, end to end: connect, create, an update guarded by --if-revision +// (both the matching case and the stale case, which must exit 13 with the +// precondition_failed body), close, dep add, ready, count, list (JSON, text +// and --deps), an external dependency through dep tree, ready, list --ready, +// ready --claim, a refused close --claim-next and a refused then forced close, +// a claim after that close, and delete. +func TestE2E_BDOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + + addr := startE2EServer(t) + workspace := t.TempDir() + + connect := runBD(t, bin, workspace, nil, + "connect", "http://"+addr, "--expect-project-id", e2eProjectID, "--json") + if connect.code != 0 { + t.Fatalf("bd connect failed (exit %d): stdout=%s stderr=%s", connect.code, connect.stdout, connect.stderr) + } + if _, err := os.Stat(filepath.Join(workspace, ".beads", "http_target.json")); err != nil { + t.Fatalf("bd connect did not write the activation sidecar: %v", err) + } + gitignore, err := os.ReadFile(filepath.Join(workspace, ".beads", ".gitignore")) + if err != nil { + t.Fatalf("bd connect did not ensure a .beads/.gitignore: %v", err) + } + if !strings.Contains(string(gitignore), "http_target.json") { + t.Errorf(".beads/.gitignore does not cover http_target.json: %s", gitignore) + } + + // create, under a named actor: bd stamps CreatedBy with the same actor it + // sends, which the http create role accepts and the server records. + const creator = "e2e-creator" + asCreator := []string{"BEADS_ACTOR=" + creator} + create := runBD(t, bin, workspace, asCreator, "create", "A title", "-d", "a description", "--json") + if create.code != 0 { + t.Fatalf("bd create failed (exit %d): stdout=%s stderr=%s", create.code, create.stdout, create.stderr) + } + id := firstJSONField(t, create.stdout, "id") + if id == "" { + t.Fatalf("bd create --json produced no id: %s", create.stdout) + } + + // read back the revision bd show reports, so the update guard below + // names a token the row actually carries. + show := runBD(t, bin, workspace, nil, "show", id, "--json") + if show.code != 0 { + t.Fatalf("bd show failed (exit %d): stderr=%s", show.code, show.stderr) + } + revision := firstJSONField(t, show.stdout, "revision") + if revision == "" { + t.Fatalf("bd show --json reported no revision: %s", show.stdout) + } + if got := firstJSONField(t, show.stdout, "created_by"); got != creator { + t.Errorf("created_by = %q after an http create, want the creating actor %q: %s", got, creator, show.stdout) + } + + // The same create routed from this http workspace into a local one + // (--repo) is written by the local store, which records whatever stamp + // the command built: it must still be the actor, since nothing about the + // workspace a command runs in changes who created the issue. + local := t.TempDir() + routed := runBD(t, bin, workspace, asCreator, "create", "Routed", "--repo", local, "--json") + if routed.code != 0 { + t.Fatalf("bd create --repo %s failed (exit %d): stdout=%s stderr=%s", local, routed.code, routed.stdout, routed.stderr) + } + routedID := firstJSONField(t, routed.stdout, "id") + if routedID == "" { + t.Fatalf("bd create --repo --json produced no id: %s", routed.stdout) + } + routedShow := runBD(t, bin, local, nil, "show", routedID, "--json") + if routedShow.code != 0 { + t.Fatalf("bd show %s in the routed-to workspace failed (exit %d): stderr=%s", routedID, routedShow.code, routedShow.stderr) + } + if got := firstJSONField(t, routedShow.stdout, "created_by"); got != creator { + t.Errorf("created_by = %q on a create routed from an http workspace, want the creating actor %q: %s", got, creator, routedShow.stdout) + } + + // update with a MATCHING --if-revision guard: must succeed. + okUpdate := runBD(t, bin, workspace, nil, "update", id, "-a", "alice", "--if-revision", revision, "--json") + if okUpdate.code != 0 { + t.Fatalf("bd update --if-revision %s (matching) failed (exit %d): stdout=%s stderr=%s", + revision, okUpdate.code, okUpdate.stdout, okUpdate.stderr) + } + + // update again with the SAME (now stale) token: must refuse with exit 13 + // and the precondition_failed machine body on the last stderr line. + staleUpdate := runBD(t, bin, workspace, nil, "update", id, "-a", "bob", "--if-revision", revision, "--json") + if staleUpdate.code != 13 { + t.Fatalf("bd update --if-revision %s (stale) exit = %d, want 13 (ExitGuardMismatch): stdout=%s stderr=%s", + revision, staleUpdate.code, staleUpdate.stdout, staleUpdate.stderr) + } + body := lastJSONLine(t, staleUpdate.stderr) + if body["code"] != "precondition_failed" { + t.Errorf("stale --if-revision body code = %v, want \"precondition_failed\": %s", body["code"], staleUpdate.stderr) + } + if body["id"] != id { + t.Errorf("stale --if-revision body id = %v, want %q: %s", body["id"], id, staleUpdate.stderr) + } + + // dep add + a second issue to depend on, then ready/count/close/delete. + // id depends on (is blocked by) depID (dep.go's own doc: "bd dep add + // issue-123 issue-456" means issue-123 depends on issue-456), so depID + // is the one issue the ready set must show and id is the one it must not. + dep := runBD(t, bin, workspace, nil, "create", "A dependency", "--json") + if dep.code != 0 { + t.Fatalf("bd create (dependency) failed (exit %d): stderr=%s", dep.code, dep.stderr) + } + depID := firstJSONField(t, dep.stdout, "id") + + if r := runBD(t, bin, workspace, nil, "dep", "add", id, depID, "--json"); r.code != 0 { + t.Fatalf("bd dep add failed (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + + ready := runBD(t, bin, workspace, nil, "ready", "--json") + if ready.code != 0 { + t.Fatalf("bd ready failed (exit %d): stdout=%s stderr=%s", ready.code, ready.stdout, ready.stderr) + } + readyIDs := jsonIDSet(t, ready.stdout) + if !readyIDs[depID] { + t.Errorf("bd ready did not list %s (no deps of its own, open): %s", depID, ready.stdout) + } + if readyIDs[id] { + t.Errorf("bd ready listed %s, which depends on the still-open %s: %s", id, depID, ready.stdout) + } + + count := runBD(t, bin, workspace, nil, "count", "--json") + if count.code != 0 { + t.Fatalf("bd count failed (exit %d): stdout=%s stderr=%s", count.code, count.stdout, count.stderr) + } + if n := jsonIntField(t, count.stdout, "count"); n != 2 { + t.Errorf("bd count = %d, want 2 (id and depID, the only issues this workspace has): %s", n, count.stdout) + } + + // The http backend serves listing, trees, claims and closes only as roles, + // and refuses the legacy store methods the external-dependency decorator + // once rebuilt those roles over, so each command below exited 1 here, even + // with nothing externally blocked. + list := runBD(t, bin, workspace, nil, "list", "--json") + if list.code != 0 { + t.Fatalf("bd list --json failed (exit %d): stdout=%s stderr=%s", list.code, list.stdout, list.stderr) + } + if listIDs := jsonIDSet(t, list.stdout); !listIDs[id] || !listIDs[depID] { + t.Errorf("bd list --json did not list both %s and %s: %s", id, depID, list.stdout) + } + textList := runBD(t, bin, workspace, nil, "list") + if textList.code != 0 || !strings.Contains(textList.stdout, id) || !strings.Contains(textList.stdout, depID) { + t.Errorf("bd list (exit %d) did not list both %s and %s: stdout=%s stderr=%s", textList.code, id, depID, textList.stdout, textList.stderr) + } + // --deps turns a failed whole-workspace edge scan behind that tree (the + // divergence ledger's L8) from a silent omission into an error. + depsList := runBD(t, bin, workspace, nil, "list", "--deps") + if depsList.code != 0 || !strings.Contains(depsList.stdout, id) || !strings.Contains(depsList.stdout, depID) { + t.Errorf("bd list --deps (exit %d) did not list both %s and %s: stdout=%s stderr=%s", depsList.code, id, depID, depsList.stdout, depsList.stderr) + } + + // An external dependency on a project this workspace has not configured + // stays unsatisfied, so extID is blocked for as long as it is open. At + // priority 0 it sorts ahead of depID, where a claim blind to that would + // take it. + const externalRef = "external:otherproj:cap" + ext := runBD(t, bin, workspace, nil, "create", "Externally blocked", "-p", "0", "--json") + if ext.code != 0 { + t.Fatalf("bd create (externally blocked) failed (exit %d): stderr=%s", ext.code, ext.stderr) + } + extID := firstJSONField(t, ext.stdout, "id") + if r := runBD(t, bin, workspace, nil, "dep", "add", extID, externalRef, "--json"); r.code != 0 { + t.Fatalf("bd dep add %s %s failed (exit %d): stdout=%s stderr=%s", extID, externalRef, r.code, r.stdout, r.stderr) + } + tree := runBD(t, bin, workspace, nil, "dep", "tree", extID, "--json") + if tree.code != 0 { + t.Fatalf("bd dep tree %s failed (exit %d): stdout=%s stderr=%s", extID, tree.code, tree.stdout, tree.stderr) + } + if !jsonIDSet(t, tree.stdout)[externalRef] { + t.Errorf("bd dep tree %s did not show its %s leaf: %s", extID, externalRef, tree.stdout) + } + for _, args := range [][]string{{"ready", "--json"}, {"list", "--ready", "--json"}} { + r := runBD(t, bin, workspace, nil, args...) + if r.code != 0 { + t.Fatalf("bd %s failed (exit %d): stdout=%s stderr=%s", strings.Join(args, " "), r.code, r.stdout, r.stderr) + } + if readyIDs := jsonIDSet(t, r.stdout); !readyIDs[depID] || readyIDs[extID] || readyIDs[id] { + t.Errorf("bd %s = %s, want %s and neither %s nor %s", strings.Join(args, " "), r.stdout, depID, extID, id) + } + } + claim := runBD(t, bin, workspace, nil, "ready", "--claim", "--json") + if claim.code != 0 { + t.Fatalf("bd ready --claim failed (exit %d): stdout=%s stderr=%s", claim.code, claim.stdout, claim.stderr) + } + if got := firstJSONField(t, claim.stdout, "id"); got != depID { + t.Errorf("bd ready --claim claimed %q, want %s (%s is externally blocked): %s", got, depID, extID, claim.stdout) + } + // --claim-next refuses whole over http, before anything closes, and names + // the commands that do the same work. It is asked again once extID is + // closed, because a closed holder of an external ref changes nothing. + refuseClaimNext := func(when string) { + t.Helper() + r := runBD(t, bin, workspace, nil, "close", depID, "--claim-next") + if r.code == 0 || !strings.Contains(r.stderr, "nothing was closed") || !strings.Contains(r.stderr, "bd ready --claim") { + t.Errorf("bd close %s --claim-next %s (exit %d) did not refuse with the bd ready --claim hint: stdout=%s stderr=%s", depID, when, r.code, r.stdout, r.stderr) + } + show := runBD(t, bin, workspace, nil, "show", depID, "--json") + if status := firstJSONField(t, show.stdout, "status"); show.code != 0 || status == "closed" { + t.Errorf("bd show %s (exit %d) status = %q after a refused bd close --claim-next %s, want it not closed: stderr=%s", depID, show.code, status, when, show.stderr) + } + } + refuseClaimNext("while " + extID + " is open") + refused := runBD(t, bin, workspace, nil, "close", extID) + if refused.code == 0 || !strings.Contains(refused.stderr, externalRef) || !strings.Contains(refused.stderr, "use --force") { + t.Errorf("bd close %s (exit %d) did not refuse it as blocked by %s: stdout=%s stderr=%s", extID, refused.code, externalRef, refused.stdout, refused.stderr) + } + if r := runBD(t, bin, workspace, nil, "close", extID, "--force", "--json"); r.code != 0 { + t.Fatalf("bd close %s --force failed (exit %d): stdout=%s stderr=%s", extID, r.code, r.stdout, r.stderr) + } + forcedShow := runBD(t, bin, workspace, nil, "show", extID, "--json") + if status := firstJSONField(t, forcedShow.stdout, "status"); forcedShow.code != 0 || status != "closed" { + t.Errorf("bd show %s (exit %d) status = %q after bd close --force, want %q: stderr=%s", extID, forcedShow.code, status, "closed", forcedShow.stderr) + } + refuseClaimNext("after " + extID + " was closed") + + // The closed extID still holds an unsatisfied external ref, so the claim + // stays on the decorator's composed leg (ledger row L14). It must still + // take what the served claim would: the one open, unassigned issue, since + // depID is claimed and id is assigned. + next := runBD(t, bin, workspace, nil, "create", "Claimable after the holder closed", "--json") + if next.code != 0 { + t.Fatalf("bd create (claimable) failed (exit %d): stdout=%s stderr=%s", next.code, next.stdout, next.stderr) + } + nextID := firstJSONField(t, next.stdout, "id") + reclaim := runBD(t, bin, workspace, nil, "ready", "--claim", "--json") + if reclaim.code != 0 { + t.Fatalf("bd ready --claim after %s was closed failed (exit %d): stdout=%s stderr=%s", extID, reclaim.code, reclaim.stdout, reclaim.stderr) + } + if got := firstJSONField(t, reclaim.stdout, "id"); got != nextID { + t.Errorf("bd ready --claim after %s was closed claimed %q, want %s: %s", extID, got, nextID, reclaim.stdout) + } + + if r := runBD(t, bin, workspace, nil, "close", depID, "--json"); r.code != 0 { + t.Fatalf("bd close failed (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + closedShow := runBD(t, bin, workspace, nil, "show", depID, "--json") + if closedShow.code != 0 { + t.Fatalf("bd show %s (after close) failed (exit %d): stderr=%s", depID, closedShow.code, closedShow.stderr) + } + if status := firstJSONField(t, closedShow.stdout, "status"); status != "closed" { + t.Errorf("bd show %s status = %q after bd close, want %q: %s", depID, status, "closed", closedShow.stdout) + } + + if r := runBD(t, bin, workspace, nil, "delete", id, "--force", "--json"); r.code != 0 { + t.Fatalf("bd delete failed (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + deletedShow := runBD(t, bin, workspace, nil, "show", id, "--json") + if deletedShow.code == 0 { + t.Errorf("bd show %s succeeded (exit 0) after bd delete --force; want a not-found refusal: stdout=%s", id, deletedShow.stdout) + } +} + +// TestE2E_ConnectRefusesPlainHTTPToNonLoopback kills the mutation that +// disables connect.go's plaintext-host refusal (`&& !connectAllowPlaintext` +// widened to `&& !connectAllowPlaintext && false`). A bearer credential this +// workspace would send to a non-loopback http:// server travels the network +// unencrypted, so connect must refuse before ever dialing — which this test +// also verifies directly: 198.51.100.1 is RFC 5737 TEST-NET-2, reserved for +// documentation and globally unroutable, so even a mutated build that skipped +// the refusal could not reach a real server here, keeping this test on the +// loopback-only network the rest of this suite requires. +func TestE2E_ConnectRefusesPlainHTTPToNonLoopback(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + workspace := t.TempDir() + + const nonLoopbackURL = "http://198.51.100.1:1/" + refused := runBD(t, bin, workspace, nil, "connect", nonLoopbackURL, "--json") + if refused.code == 0 { + t.Fatalf("bd connect %s succeeded; want a refusal (not loopback, plain http, no --allow-plaintext): stdout=%s", nonLoopbackURL, refused.stdout) + } + if !strings.Contains(refused.stderr, "not loopback") { + t.Errorf("bd connect %s stderr does not name the loopback refusal; want the host-safety gate, not some other failure: exit=%d stdout=%s stderr=%s", + nonLoopbackURL, refused.code, refused.stdout, refused.stderr) + } + if _, err := os.Stat(filepath.Join(workspace, ".beads", "http_target.json")); err == nil { + t.Error("bd connect wrote the activation sidecar despite refusing the connection") + } +} + +// TestE2E_ConnectRefusesSwitchingBackendWithoutForce kills the mutation that +// disables connect.go's backend-switch force gate (`&& !connectForce` +// widened to `&& !connectForce && false`). A workspace that already selects +// a different backend must refuse to switch to http without --force; with +// --force it must proceed PAST that gate, so the test tells the two +// refusals apart by what they say, not just that they both fail: a port +// nothing listens on (still loopback, so it clears the plaintext-host gate +// above) turns "the force gate let this through" into its own distinct +// dial failure rather than a false pass. +func TestE2E_ConnectRefusesSwitchingBackendWithoutForce(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + workspace := t.TempDir() + beadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt"}`), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + + const target = "http://127.0.0.1:1/" + + withoutForce := runBD(t, bin, workspace, nil, "connect", target, "--json") + if withoutForce.code == 0 { + t.Fatalf("bd connect %s without --force succeeded; workspace already selects backend %q", target, "dolt") + } + if !strings.Contains(withoutForce.stderr, "--force") { + t.Errorf("bd connect without --force did not mention --force in its refusal; want the backend-switch gate, not some other failure: exit=%d stdout=%s stderr=%s", + withoutForce.code, withoutForce.stdout, withoutForce.stderr) + } + + withForce := runBD(t, bin, workspace, nil, "connect", target, "--force", "--json") + if withForce.code == 0 { + t.Fatalf("bd connect %s --force succeeded against a port nothing listens on", target) + } + if strings.Contains(withForce.stderr, "--force") { + t.Errorf("bd connect --force still hit the force-gate refusal; the gate was not bypassed: stderr=%s", withForce.stderr) + } + + data, err := os.ReadFile(filepath.Join(beadsDir, "metadata.json")) + if err != nil { + t.Fatalf("read metadata.json: %v", err) + } + if !strings.Contains(string(data), `"dolt"`) { + t.Errorf("metadata.json no longer names backend dolt after the refused connect attempt: %s", data) + } +} + +// TestE2E_ConnectRedactsCredentialsEverywhereItPrintsTheURL is MED-3's +// regression test: a URL pasted with a userinfo password must never have +// that password echoed back by connect, in any output stream, on any path — +// success or failure. The target here fails at the handshake dial (loopback, +// nothing listening on port 1), which is exactly the path that used to wrap +// the raw argument into "connecting to %s: %v". +func TestE2E_ConnectRedactsCredentialsEverywhereItPrintsTheURL(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + workspace := t.TempDir() + + const password = "s3cr3t-password-998" //nolint — a fixture value, not a real credential. + target := "http://alice:" + password + "@127.0.0.1:1/" + + r := runBD(t, bin, workspace, nil, "connect", target, "--json") + if r.code == 0 { + t.Fatalf("bd connect %s succeeded against a port nothing listens on", target) + } + if strings.Contains(r.stdout, password) || strings.Contains(r.stderr, password) { + t.Errorf("bd connect's output carries the literal userinfo password %q: stdout=%s stderr=%s", password, r.stdout, r.stderr) + } +} + +// TestE2E_ReconnectRefusesADifferentProjectUnlessForced is MED-5's test: a +// bare re-connect (no --expect-project-id, no --force) against a server that +// answers with a DIFFERENT project id than the one the workspace's sidecar +// already pinned must refuse, not silently re-pin — connect.go used to build +// the new Target from --expect-project-id alone, so a plain `bd connect +// ` with nothing on the command line would adopt whatever the +// new server said with no refusal at all. --force must still get past it. +func TestE2E_ReconnectRefusesADifferentProjectUnlessForced(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + + firstAddr := startE2EServer(t) // publishes e2eProjectID + secondAddr := startE2EServerWithIdentity(t, "proj-e2e-s6-OTHER", "httpe2e_other") + workspace := t.TempDir() + + first := runBD(t, bin, workspace, nil, "connect", "http://"+firstAddr, "--json") + if first.code != 0 { + t.Fatalf("first bd connect failed (exit %d): stdout=%s stderr=%s", first.code, first.stdout, first.stderr) + } + pinned, err := os.ReadFile(filepath.Join(workspace, ".beads", "http_target.json")) + if err != nil { + t.Fatalf("read sidecar after first connect: %v", err) + } + if !strings.Contains(string(pinned), e2eProjectID) { + t.Fatalf("sidecar after first connect does not carry %q: %s", e2eProjectID, pinned) + } + + // A bare re-connect to the DIFFERENT server, no --expect-project-id, no + // --force: must refuse, carrying the prior pin forward rather than + // adopting the new server's answer. + again := runBD(t, bin, workspace, nil, "connect", "http://"+secondAddr, "--json") + if again.code == 0 { + t.Fatalf("bare re-connect to a different project's server succeeded; want a refusal: stdout=%s", again.stdout) + } + if !strings.Contains(again.stderr, e2eProjectID) { + t.Errorf("re-connect refusal does not name the prior pin %q: stderr=%s", e2eProjectID, again.stderr) + } + stillPinned, err := os.ReadFile(filepath.Join(workspace, ".beads", "http_target.json")) + if err != nil { + t.Fatalf("read sidecar after refused re-connect: %v", err) + } + if string(stillPinned) != string(pinned) { + t.Errorf("sidecar changed after a REFUSED re-connect: before=%s after=%s", pinned, stillPinned) + } + + // --force gets past it and re-pins to the new server's project. + forced := runBD(t, bin, workspace, nil, "connect", "http://"+secondAddr, "--force", "--json") + if forced.code != 0 { + t.Fatalf("bd connect --force to the other project failed (exit %d): stdout=%s stderr=%s", forced.code, forced.stdout, forced.stderr) + } + refetched, err := os.ReadFile(filepath.Join(workspace, ".beads", "http_target.json")) + if err != nil { + t.Fatalf("read sidecar after forced re-connect: %v", err) + } + if !strings.Contains(string(refetched), "proj-e2e-s6-OTHER") { + t.Errorf("sidecar after --force re-connect does not carry the new project id: %s", refetched) + } +} + +// TestE2E_SwitchingBackToDoltLeavesOriginalIdentityIntact is MED-6's test: +// `bd connect --force` on a workspace that already selects dolt must not +// rewrite that workspace's git-tracked metadata.json project_id/database — +// the http identity belongs in the gitignored sidecar alone. This proves it +// by round-tripping: a dolt workspace with its own original project_id and +// database connects to http, and metadata.json's original dolt fields must +// still read back unchanged afterward — exactly what "switching back to +// dolt" would see, since nothing ever touched them to switch back FROM. +func TestE2E_SwitchingBackToDoltLeavesOriginalIdentityIntact(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) // publishes e2eProjectID, unrelated to the dolt identity below. + workspace := t.TempDir() + beadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + const ( + originalProjectID = "dolt-original-project" + originalDatabase = "dolt-original-db" + ) + metadataPath := filepath.Join(beadsDir, "metadata.json") + original := fmt.Sprintf(`{"backend":"dolt","database":%q,"project_id":%q}`, originalDatabase, originalProjectID) + if err := os.WriteFile(metadataPath, []byte(original), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + + forced := runBD(t, bin, workspace, nil, "connect", "http://"+addr, "--force", "--json") + if forced.code != 0 { + t.Fatalf("bd connect --force failed (exit %d): stdout=%s stderr=%s", forced.code, forced.stdout, forced.stderr) + } + + data, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatalf("read metadata.json after connect: %v", err) + } + var after struct { + Backend string `json:"backend"` + Database string `json:"database"` + ProjectID string `json:"project_id"` + } + if err := json.Unmarshal(data, &after); err != nil { + t.Fatalf("parse metadata.json after connect: %v\n%s", err, data) + } + if after.Backend != "http" { + t.Errorf("metadata.json backend = %q after connect, want %q: %s", after.Backend, "http", data) + } + // The MED-6 assertion: connect must NOT have overwritten these with the + // http server's own database/project_id (e2eDatabase/e2eProjectID) — a + // future switch back to dolt reads these same, untouched fields. + if after.Database != originalDatabase { + t.Errorf("metadata.json database = %q after bd connect --force, want the original %q untouched (MED-6: connect must not rewrite it): %s", + after.Database, originalDatabase, data) + } + if after.ProjectID != originalProjectID { + t.Errorf("metadata.json project_id = %q after bd connect --force, want the original %q untouched (MED-6: connect must not rewrite it): %s", + after.ProjectID, originalProjectID, data) + } + + // The http identity itself lives in the sidecar, not metadata.json. + sidecar, err := os.ReadFile(filepath.Join(beadsDir, "http_target.json")) + if err != nil { + t.Fatalf("read sidecar after connect: %v", err) + } + if !strings.Contains(string(sidecar), e2eProjectID) { + t.Errorf("sidecar does not carry the http server's project id %q: %s", e2eProjectID, sidecar) + } +} + +// TestE2E_ConnectClearRestoresThePreviousBackend pins `bd connect --clear` +// (bee-ghosttrack CHANGES_REQUESTED on #7288, should-fix 1): a dolt workspace +// forced onto http and then cleared must select dolt again, with the sidecar +// gone and metadata.json's own dolt identity untouched. A second --clear finds +// nothing to clear and leaves metadata.json alone. +func TestE2E_ConnectClearRestoresThePreviousBackend(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + workspace := t.TempDir() + beadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + const ( + originalProjectID = "dolt-original-project-for-clear" + originalDatabase = "dolt-original-db-for-clear" + ) + metadataPath := filepath.Join(beadsDir, "metadata.json") + original := fmt.Sprintf(`{"backend":"dolt","database":%q,"project_id":%q}`, originalDatabase, originalProjectID) + if err := os.WriteFile(metadataPath, []byte(original), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + sidecarPath := filepath.Join(beadsDir, "http_target.json") + + forced := runBD(t, bin, workspace, nil, "connect", "http://"+addr, "--force", "--json") + if forced.code != 0 { + t.Fatalf("bd connect --force failed (exit %d): stdout=%s stderr=%s", forced.code, forced.stdout, forced.stderr) + } + if _, err := os.Stat(sidecarPath); err != nil { + t.Fatalf("bd connect --force did not write the sidecar: %v", err) + } + + type clearResult struct { + Cleared bool `json:"cleared"` + RestoredBackend string `json:"restored_backend"` + } + type identity struct { + Backend string `json:"backend"` + Database string `json:"database"` + ProjectID string `json:"project_id"` + } + readMetadata := func() identity { + t.Helper() + data, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatalf("read metadata.json: %v", err) + } + var got identity + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("parse metadata.json: %v\n%s", err, data) + } + return got + } + want := identity{Backend: "dolt", Database: originalDatabase, ProjectID: originalProjectID} + + cleared := runBD(t, bin, workspace, nil, "connect", "--clear", "--json") + if cleared.code != 0 { + t.Fatalf("bd connect --clear failed (exit %d): stdout=%s stderr=%s", cleared.code, cleared.stdout, cleared.stderr) + } + var first clearResult + if err := json.Unmarshal([]byte(cleared.stdout), &first); err != nil { + t.Fatalf("parse bd connect --clear --json: %v\n%s", err, cleared.stdout) + } + if !first.Cleared || first.RestoredBackend != "dolt" { + t.Errorf("bd connect --clear = %+v, want cleared and dolt restored: %s", first, cleared.stdout) + } + if _, err := os.Stat(sidecarPath); !os.IsNotExist(err) { + t.Errorf("sidecar still present after bd connect --clear (stat err %v)", err) + } + if got := readMetadata(); got != want { + t.Errorf("metadata.json after bd connect --clear = %+v, want %+v", got, want) + } + + again := runBD(t, bin, workspace, nil, "connect", "--clear", "--json") + if again.code != 0 { + t.Fatalf("second bd connect --clear failed (exit %d): stdout=%s stderr=%s", again.code, again.stdout, again.stderr) + } + var second clearResult + if err := json.Unmarshal([]byte(again.stdout), &second); err != nil { + t.Fatalf("parse second bd connect --clear --json: %v\n%s", err, again.stdout) + } + if second != (clearResult{}) { + t.Errorf("second bd connect --clear = %+v, want nothing cleared or restored: %s", second, again.stdout) + } + if got := readMetadata(); got != want { + t.Errorf("metadata.json after a second bd connect --clear = %+v, want %+v", got, want) + } +} + +// TestE2E_ForceConnectWritesSucceedDespiteDifferingLocalProjectID pins the +// bee-ghosttrack CHANGES_REQUESTED finding on #7288: after `bd connect +// --force` over a workspace that already had a LOCAL (dolt) identity, every +// write used to fail with "workspace identity mismatch detected". That +// check compared metadata.json's project_id (the original dolt workspace's +// identity, left untouched by connect per MED-6 — see +// TestE2E_SwitchingBackToDoltLeavesOriginalIdentityIntact above) against the +// live server's _project_id, and those two are legitimately different +// projects; the check needs the http backend's OWN pin (the sidecar's +// ExpectProjectID, set by this same connect) instead. +// +// This reuses that test's exact setup — a workspace with its own original +// dolt project_id, forced onto an unrelated server's project — but goes +// further: it actually writes (create, update, comments add) over the new +// http connection, which is what the finding says used to fail outright. +func TestE2E_ForceConnectWritesSucceedDespiteDifferingLocalProjectID(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) // publishes e2eProjectID, unrelated to the dolt identity below. + workspace := t.TempDir() + beadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + const ( + originalProjectID = "dolt-original-project-for-writes" + originalDatabase = "dolt-original-db-for-writes" + ) + metadataPath := filepath.Join(beadsDir, "metadata.json") + original := fmt.Sprintf(`{"backend":"dolt","database":%q,"project_id":%q}`, originalDatabase, originalProjectID) + if err := os.WriteFile(metadataPath, []byte(original), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + + forced := runBD(t, bin, workspace, nil, "connect", "http://"+addr, "--force", "--json") + if forced.code != 0 { + t.Fatalf("bd connect --force failed (exit %d): stdout=%s stderr=%s", forced.code, forced.stdout, forced.stderr) + } + + create := runBD(t, bin, workspace, nil, "create", "A title over http after forced connect", "--json") + if create.code != 0 { + t.Fatalf("bd create after forced connect failed (exit %d): stdout=%s stderr=%s", create.code, create.stdout, create.stderr) + } + if strings.Contains(create.stderr, "workspace identity mismatch") { + t.Fatalf("bd create tripped the workspace-identity-mismatch refusal despite connect having just pinned this project: stderr=%s", create.stderr) + } + id := firstJSONField(t, create.stdout, "id") + if id == "" { + t.Fatalf("bd create --json produced no id: %s", create.stdout) + } + + update := runBD(t, bin, workspace, nil, "update", id, "-a", "alice", "--json") + if update.code != 0 { + t.Fatalf("bd update after forced connect failed (exit %d): stdout=%s stderr=%s", update.code, update.stdout, update.stderr) + } + if strings.Contains(update.stderr, "workspace identity mismatch") { + t.Fatalf("bd update tripped the workspace-identity-mismatch refusal despite connect having just pinned this project: stderr=%s", update.stderr) + } + + comment := runBD(t, bin, workspace, nil, "comments", "add", id, "a comment over http after forced connect", "--json") + if comment.code != 0 { + t.Fatalf("bd comments add after forced connect failed (exit %d): stdout=%s stderr=%s", comment.code, comment.stdout, comment.stderr) + } + if strings.Contains(comment.stderr, "workspace identity mismatch") { + t.Fatalf("bd comments add tripped the workspace-identity-mismatch refusal despite connect having just pinned this project: stderr=%s", comment.stderr) + } +} + +// startE2EServer boots an in-process `bd serve` over embedded Dolt and +// returns its bound 127.0.0.1 address. It is the same role-binding shape as +// internal/httpclient's servedEnv harness, kept separate because that one is +// unexported in a different package: this test drives the CLI, not the +// internal wire client, so it owns its own (smaller) copy rather than +// reaching across the package boundary. +// +// This stays the one server topology this file uses — a real `bd serve` +// SUBPROCESS, gated on BEADS_TEST_DOLT_SERVER=local the way +// internal/testutil's local Dolt backend is elsewhere, was considered and +// rejected: +// +// 1. bd serve permanently refuses an embedded-Dolt workspace +// (cmd/bd/serve.go's serveDatabaseSource, "EMBEDDED DOLT IS PERMANENT"). +// A real `bd` subprocess pointed at a `bd init`-created workspace with no +// registered backend would therefore refuse to start at all — there is no +// bd-init-then-bd-serve shape for this backend to drive as a subprocess. +// 2. The one way around that refusal, a registered backend whose Open hands +// back a real embedded store (what cmd/bd/serve_registered_backend_test.go +// does), only exists as Go wiring injected into the SAME test process — +// registration happens at init() time, so a spawned `bd` subprocess has +// nothing registered and cannot reproduce it. That file's own comment +// reaches the identical conclusion for the identical reason. +// 3. The other way around it is a real, non-embedded Dolt SQL server behind +// "shared"/proxied mode (BEADS_DOLT_SHARED_SERVER=1, internal/testutil's +// local-or-container harness, cmd/bd/serve_proxied_integration_test.go's +// startServe). That subprocess `bd serve` lifecycle contract — the bound +// address on stdout's first line, signal-driven shutdown, the auth +// posture warnings — is already pinned end to end there, against exactly +// that topology. Reproducing it a second time in this package would +// stand up a second real Dolt SQL server fixture per run to re-test the +// same `bd serve` CLI plumbing that package already owns, not anything +// specific to the public backend/http door this package exists to prove. +// +// What this e2e tier specifically has to show — bd connect, then full CRUD, +// actually working end to end over the wire against a real listening server +// — is exactly what the in-process httpapi.Listen harness below proves: it +// is the identical httpapi.Config / Listen / Serve call bd serve's own +// serveListen makes, bound to a real TCP socket a real `bd` binary dials +// unmodified. A real `bd serve` subprocess on top of that would add only +// coverage of serve's own flag parsing and workspace-source resolution, +// which belongs to (and is already owned by) cmd/bd/serve_test.go and +// cmd/bd/serve_proxied_integration_test.go. +func startE2EServer(t *testing.T) string { + t.Helper() + return startE2EServerWithIdentity(t, e2eProjectID, e2eDatabase) +} + +// startE2EServerWithIdentity is startE2EServer, parameterized on the identity +// the in-process server publishes at handshake. MED-5's reconnect-mismatch +// test needs a SECOND server that answers with a DIFFERENT project id than +// the one a prior `bd connect` already pinned, to prove the pin is honored +// rather than silently replaced. +func startE2EServerWithIdentity(t *testing.T, projectID, database string) string { + t.Helper() + beadsDir := t.TempDir() + ctx := context.Background() + reference, err := embeddeddolt.Open(ctx, beadsDir, database, "main") + if err != nil { + t.Fatalf("open the reference store: %v", err) + } + t.Cleanup(func() { _ = reference.Close() }) + if err := reference.SetConfig(ctx, "issue_prefix", "e2e"); err != nil { + t.Fatalf("set the issue prefix: %v", err) + } + + cfg := e2eServeConfig(t, reference, projectID, database) + srv, err := httpapi.Listen(cfg) + if err != nil { + t.Fatalf("bind the in-process server: %v", err) + } + serveCtx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = srv.Serve(serveCtx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Error("the in-process server did not shut down") + } + }) + return srv.Addr() +} + +func e2eServeConfig(t *testing.T, s storage.DoltStorage, projectID, database string) httpapi.Config { + t.Helper() + cfg := httpapi.Config{ + Addr: "127.0.0.1:0", + Stdout: io.Discard, + Stderr: io.Discard, + Workspace: domain.ContextInfo{ProjectID: projectID, Database: database}, + } + var err error + fail := func(name string) { + t.Helper() + if err != nil { + t.Fatalf("%s(): %v", name, err) + } + } + cfg.Reader, err = s.IssueReader() + fail("IssueReader") + cfg.Claimer, err = s.IssueClaimer() + fail("IssueClaimer") + cfg.ReadyClaimer, err = s.ReadyClaimer() + fail("ReadyClaimer") + cfg.Releaser, err = s.Releaser() + fail("Releaser") + cfg.Lifecycle, err = s.IssueLifecycle() + fail("IssueLifecycle") + cfg.Settings, err = s.WorkspaceConfig() + fail("WorkspaceConfig") + cfg.Stats, err = s.StatsReporter() + fail("StatsReporter") + cfg.CycleDetector, err = s.CycleDetector() + fail("CycleDetector") + cfg.EdgeReader, err = s.EdgeReader() + fail("EdgeReader") + cfg.GraphCounter, err = s.GraphCounter() + fail("GraphCounter") + cfg.BatchGetter, err = s.BatchGetter() + fail("BatchGetter") + cfg.LeaseReclaimer, err = s.LeaseReclaimer() + fail("LeaseReclaimer") + cfg.Relations, err = s.IssueRelations() + fail("IssueRelations") + cfg.Commenter, err = s.Commenter() + fail("Commenter") + cfg.BlockingAnnotator, err = s.BlockingAnnotator() + fail("BlockingAnnotator") + cfg.TreeWalker, err = s.TreeWalker() + fail("TreeWalker") + cfg.ReadyCounter, err = s.ReadyCounter() + fail("ReadyCounter") + cfg.Counter, err = s.Counter() + fail("Counter") + cfg.Querier, err = s.Querier() + fail("Querier") + cfg.Sweeper, err = s.Sweeper() + fail("Sweeper") + cfg.Deleter, err = s.Deleter() + fail("Deleter") + cfg.BatchCreator, err = s.BatchCreator() + fail("BatchCreator") + cfg.BatchCloser, err = s.BatchCloser() + fail("BatchCloser") + cfg.DependencyEditor, err = s.DependencyEditor() + fail("DependencyEditor") + cfg.MetadataCAS, err = s.MetadataCAS() + fail("MetadataCAS") + cfg.BatchApplier, err = s.BatchApplier() + fail("BatchApplier") + cfg.Memories, err = s.Memories() + fail("Memories") + return cfg +} + +// skipUnlessEmbeddedDolt mirrors internal/httpclient's gate: this tier needs +// the cgo-linked embedded engine, and BEADS_HTTP_TEST_REQUIRED=1 is how the +// served-surface lane says that tier is mandatory rather than best-effort. +func skipUnlessEmbeddedDolt(t *testing.T) { + t.Helper() + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") == "1" { + return + } + if os.Getenv("BEADS_HTTP_TEST_REQUIRED") == "1" { + t.Fatal("BEADS_HTTP_TEST_REQUIRED=1 but BEADS_TEST_EMBEDDED_DOLT is not 1; the http e2e tier is not enforced") + } + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run the http backend e2e tier") +} + +type bdResult struct { + stdout, stderr string + code int +} + +// runBD shells the built bd binary out in dir, the same subprocess pattern +// test/conformance's e2e runner uses, kept local here so this tier has no +// dependency on that package's unexported helpers. +func runBD(t *testing.T, bin, dir string, env []string, args ...string) bdResult { + t.Helper() + cmd := exec.Command(bin, args...) + cmd.Dir = dir + cmd.Env = minimalBDEnv(dir, env...) + var o, e strings.Builder + cmd.Stdout, cmd.Stderr = &o, &e + err := cmd.Run() + code := 0 + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + code = ee.ExitCode() + } else { + t.Fatalf("running bd %s: %v", strings.Join(args, " "), err) + } + } + return bdResult{stdout: o.String(), stderr: e.String(), code: code} +} + +// minimalBDEnv builds a hermetic environment for a bd subprocess, the same +// sanitizing shape cmd/bd's own proxied-integration harness uses +// (bdProxiedEnv): PATH plus the few variables a Go binary needs to start, +// HOME repointed at dir rather than the real one, and nothing BEADS_* +// carried over from whatever invoked `go test`. A stray BEADS_DIR or +// BEADS_HTTP_TOKEN* exported in this process's own environment must never +// leak into a workspace this test means to fully control — this harness +// ran the full os.Environ() through unfiltered before, which a credential +// or workspace-override variable set in the ambient shell could silently +// have reached. extra layers on top (a test case's own deliberate override), +// so it is applied last and wins any collision. +func minimalBDEnv(dir string, extra ...string) []string { + env := []string{ + "HOME=" + dir, + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + } + for _, key := range []string{"PATH", "TMPDIR", "SYSTEMROOT", "WINDIR"} { + if v, ok := os.LookupEnv(key); ok { + env = append(env, key+"="+v) + } + } + return append(env, extra...) +} + +// jsonIDSet collects every top-level "id" field out of a bd --json array +// response (bd ready among others), for the containment checks a real +// assertion needs rather than firstJSONField's single-match convenience. +func jsonIDSet(t *testing.T, out string) map[string]bool { + t.Helper() + var rows []map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &rows); err != nil { + t.Fatalf("parse JSON array output: %v\n%s", err, out) + } + ids := make(map[string]bool, len(rows)) + for _, row := range rows { + if id, ok := row["id"].(string); ok { + ids[id] = true + } + } + return ids +} + +// jsonIntField pulls one numeric top-level field out of a bd --json object +// response (bd count's {"count": N}). +func jsonIntField(t *testing.T, out, field string) int64 { + t.Helper() + var generic map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &generic); err != nil { + t.Fatalf("parse JSON output for field %q: %v\n%s", field, err, out) + } + v, ok := generic[field].(float64) + if !ok { + t.Fatalf("field %q missing or not numeric in: %s", field, out) + } + return int64(v) +} + +// firstJSONField pulls one string-valued top-level field out of a bd --json +// response, tolerating the schema_version envelope some commands wrap their +// payload in and the bare-object shape others return directly. +func firstJSONField(t *testing.T, out, field string) string { + t.Helper() + var generic any + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &generic); err != nil { + t.Fatalf("parse JSON output for field %q: %v\n%s", field, err, out) + } + return findField(generic, field) +} + +func findField(v any, field string) string { + switch m := v.(type) { + case map[string]any: + if s, ok := m[field].(string); ok { + return s + } + for _, nested := range []string{"data", "issue"} { + if inner, ok := m[nested]; ok { + if s := findField(inner, field); s != "" { + return s + } + } + } + case []any: + for _, item := range m { + if s := findField(item, field); s != "" { + return s + } + } + } + return "" +} + +// lastJSONLine parses the final non-blank line of stderr as JSON, matching +// reportIfRevisionFailure's own documented contract ("the JSON body is the +// LAST line on stderr"). +func lastJSONLine(t *testing.T, stderr string) map[string]any { + t.Helper() + lines := strings.Split(strings.TrimRight(stderr, "\n"), "\n") + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if line == "" { + continue + } + var body map[string]any + if err := json.Unmarshal([]byte(line), &body); err == nil { + return body + } + t.Fatalf("last stderr line is not JSON: %q\nfull stderr:\n%s", line, stderr) + } + t.Fatalf("stderr had no non-blank line to parse:\n%s", stderr) + return nil +} + +var ( + buildBDOnce sync.Once + bdBinPath string + bdBuildErr error + bdBinOwnBuild bool // true only on the go-build-from-source path; false for Bazel's prebuilt binary. +) + +// buildBD builds the bd binary once per test process, matching the +// gms_pure_go embedded-engine tag used throughout cmd/bd's own CLI tests +// (e.g. cmd/bd/create_deps_atomic_test.go). Under Bazel there is no module +// tree to build from, so the prebuilt test binary is used instead. +// +// The binary is shared, via buildBDOnce, across every test in this package +// that calls buildBD — that sharing is the whole point, one build instead of +// one per test. Cleanup therefore cannot be a per-call t.Cleanup: the first +// caller to win the Once race would be the first test to finish, and its +// Cleanup would delete the binary every LATER test in the same process still +// needs (exactly what happened before this comment existed — the second and +// third test to call buildBD got "no such file or directory" because the +// first test's Cleanup had already removed it). TestMain below owns deleting +// it exactly once, after every test in the package has finished with it. +func buildBD(t *testing.T) string { + t.Helper() + buildBDOnce.Do(func() { + if bazeltest.IsBazel() { + bdBinPath, bdBuildErr = bazeltest.PrebuiltBD() + return + } + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "bd-http-e2e") + if err != nil { + bdBuildErr = fmt.Errorf("mkdir temp for bd binary: %w", err) + return + } + bin := filepath.Join(dir, "bd-http-e2e") + cmd := exec.Command("go", "build", "-tags", "gms_pure_go", "-o", bin, "./cmd/bd") + cmd.Dir = e2eRepoRoot() + if out, err := cmd.CombinedOutput(); err != nil { + bdBuildErr = fmt.Errorf("build bd: %w\n%s", err, out) + _ = os.RemoveAll(dir) + return + } + bdBinPath = bin + bdBinOwnBuild = true + }) + if bdBuildErr != nil { + t.Fatal(bdBuildErr) + } + return bdBinPath +} + +// TestMain exists for exactly one reason: to delete the temp directory +// buildBD's go-build-from-source path creates, once, after every test in +// this package has had its turn with the shared binary inside it. See +// buildBD's doc comment for why a per-test t.Cleanup cannot do this job. +func TestMain(m *testing.M) { + code := m.Run() + if bdBinOwnBuild && bdBinPath != "" { + _ = os.RemoveAll(filepath.Dir(bdBinPath)) + } + os.Exit(code) +} + +func e2eRepoRoot() string { + _, file, _, _ := runtime.Caller(0) // /backend/http/e2e_cli_test.go + return filepath.Dir(filepath.Dir(filepath.Dir(file))) +} diff --git a/backend/http/errors.go b/backend/http/errors.go new file mode 100644 index 000000000..610391c94 --- /dev/null +++ b/backend/http/errors.go @@ -0,0 +1,111 @@ +package bdhttp + +import ( + "github.com/steveyegge/beads/internal/httpclient" + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// The refusal taxonomy, re-exported so an embedder can CLASSIFY what this +// backend answers rather than match its text. +// +// Every name here is an alias of the internal one, so errors.Is and +// errors.As reach the same values across the module boundary. Two audiences +// need that: the host application deciding whether a failure is the user's, +// the server's or the wiring's, and any surface that renders it, since these +// types carry the server URL, its release and its advertised capabilities +// precisely so a renderer parses nothing. + +// ErrNoTransport reports a process that registered this backend without +// installing a transport. It is a build-wiring fault, not a user's: the +// workspace and the server may both be fine. Register cannot produce it — it +// installs the transport itself — so meeting it means something reached the +// registry another way (for example a test double that calls +// backends.Register directly). +var ErrNoTransport = httpclient.ErrNoTransport + +// The store's own refusals. A method this backend does not serve returns +// UnsupportedError, which unwraps to the portable storage.ErrUnsupported, so +// the classification a caller already does for a local backend holds +// unchanged. +type ( + // UnsupportedError is what every refused method of this backend returns: a + // STRUCT, reached with errors.As, carrying the server it refused on, that + // server's release and the capabilities it does advertise. It unwraps to + // the portable storage.ErrUnsupported, so a caller that only wants "this + // backend cannot do that" matches the same sentinel it would for a local + // store and needs none of these fields. + UnsupportedError = httpclient.ErrHTTPUnsupported + // InexpressibleError is a request carrying a field the v0 wire has no + // member for. It is both an unsupported-operation refusal and an encoder + // refusal, because two audiences classify it, and errors.As reaches either + // arm. + InexpressibleError = httpclient.InexpressibleError + // PartialIDSearchError answers a substring or prefix id lookup, which no + // server operation serves. The input may be a partial id OR a full id that + // does not exist, and the client cannot tell which, so the refusal names + // both outcomes. + PartialIDSearchError = httpclient.PartialIDSearchError + // WatchRefusedError refuses a `--watch`-style polling loop, which + // amplifies load on a shared server with no change detection to justify + // it. + WatchRefusedError = httpclient.WatchRefusedError + // RedactedSettingError reports a setting whose value the server withheld + // because its key marks it credential-bearing. It is not an unset key: one + // says the workspace stores nothing, the other says the workspace stores + // something this client may not see. + RedactedSettingError = httpclient.RedactedSettingError +) + +// The class sentinels behind those types, for a caller that classifies +// without reaching for fields. +var ( + ErrPartialIDSearch = httpclient.ErrPartialIDSearch + ErrWatchUnsupported = httpclient.ErrWatchUnsupported + ErrSettingRedacted = httpclient.ErrSettingRedacted +) + +// The wire's own refusals, which reach a caller through Open, Handshake and +// every dispatch a store makes. +type ( + // ProjectMismatchError is the wrong-server diagnostic: the server + // answered, and it owns a different workspace. It names both project ids + // AND the server's own database and repo root, because "the ids differ" + // does not say WHICH wrong server answered — and on a shared host that is + // the question. + ProjectMismatchError = wire.ProjectMismatchError + // APIVersionError reports a server on a path major this client cannot + // address. It names both versions, because neither alone says which side + // to move. + APIVersionError = wire.APIVersionError + // CapabilityError reports an operation the server does not advertise. It + // is how version skew is told apart from a missing entity: an unrouted + // path on an older server answers a bare 404, indistinguishable from + // not_found, so the advertised list is consulted before the dial. + CapabilityError = wire.CapabilityError + // ProblemError is a refusal the server described: its status, its + // problem code, and the typed discriminators it carries. Unwrap reaches + // the sentinel below that the code maps to. + ProblemError = wire.ProblemError +) + +var ( + ErrProjectMismatch = wire.ErrProjectMismatch + ErrAPIVersion = wire.ErrAPIVersion + ErrCapabilityAbsent = wire.ErrCapabilityAbsent + + // ErrUnauthenticated is the credential the server refused. The error + // names which ladder rung held it — a provenance label, never the + // credential — so an operator knows what to rotate. + ErrUnauthenticated = wire.ErrUnauthenticated + ErrBadRequest = wire.ErrBadRequest + ErrInvalidCursor = wire.ErrInvalidCursor + ErrBusy = wire.ErrBusy + ErrDBUnavailable = wire.ErrDBUnavailable + ErrServerFault = wire.ErrServerFault +) + +// ProjectMismatchRecovery is the one action that resolves a wrong-server +// mismatch: point the workspace back at the server that owns it (re-run +// `bd connect`). There is no local database to reconcile, so the repair +// tooling a local backend would name does not apply. +const ProjectMismatchRecovery = wire.ProjectMismatchRecovery diff --git a/backend/http/http.go b/backend/http/http.go new file mode 100644 index 000000000..cddbbab2d --- /dev/null +++ b/backend/http/http.go @@ -0,0 +1,216 @@ +// Package bdhttp is the public door to the http client store that ships +// inside this module (internal/httpclient) — the storage backend that speaks +// the v0 `bd serve` wire to a remote server — for embedders that link beads +// as a library rather than running cmd/bd. +// +// Such an embedder cannot reach the store any other way, for the reason its +// (hypothetical) postgres sibling would document: the registry facade's +// Register takes a backends.Backend value whose constructors live in an +// internal package, and the only other production wiring is cmd/bd's. Without +// this package a workspace whose .beads/metadata.json names "http" is a +// workspace an embedder can only tombstone. +// +// # Registration is two-step, and that is why Register takes Options +// +// Putting "http" in the registry is not enough to open anything: the store +// dials through a process-wide transport constructor that registration does +// not install on its own, so a binary that registered and stopped there would +// fail every open with httpclient.ErrNoTransport — a build-wiring fault, not a +// workspace or server problem. Register does both: it adds "http" to the +// registry AND installs the default dialer, and it installs a per-open +// OpenWith hook so a multi-tenant embedder (for example gc, Gas City — one +// process serving many workspaces, called "cities") can route a distinct +// credential through backends.OpenOptions / beads.OpenBestAvailableWith for +// each workspace it opens, without the process-wide dialer being asked to +// serve two tenants with one fixed credential. +// +// # Stability +// +// EXPERIMENTAL, on the same terms as the backend package this sits beside. +// Pin an exact beads version. +package bdhttp + +import ( + "context" + "net/http" + + "github.com/steveyegge/beads/backend" + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient" + "github.com/steveyegge/beads/internal/storage/backends" +) + +// UserAgentSuffix identifies this backend inside a stamped User-Agent, so a +// server log line says both which binary and which client spoke to it. An +// embedder that stamps its own build joins the two the way cmd/bd does: +// "myapp/1.4.2 " + bdhttp.UserAgentSuffix. +const UserAgentSuffix = httpclient.WireUserAgentSuffix + +// Options configures the transport Register installs and Open/Handshake dial +// with. +// +// The zero value is usable and is what the CLI's own wiring amounts to: the +// built-in bearer credential ladder, this module's default User-Agent, and +// the standard http transport. +type Options struct { + // UserAgent identifies the embedding build on every request. Empty sends + // UserAgentSuffix alone, which names the backend but not the binary. + UserAgent string + // HTTPClient replaces the transport — a tuned timeout, a proxy, a custom + // TLS config, an httptest server in a test. Nil builds one. The client is + // used as given except for redirect handling, which the underlying wire + // client always forces off: a followed 30x would replay the Authorization + // header at whatever host the Location named. + HTTPClient *http.Client + // RequireCredential makes the registered backend's OpenWith seam refuse a + // registered open with no OpenOptions.Credential, rather than silently + // falling back to the ambient bearer ladder (BEADS_HTTP_TOKEN, + // BEADS_HTTP_TOKEN_COMMAND, the credentials file). It has no effect on + // Open/Handshake below, or on the plain Open/OpenReadOnly path a + // single-tenant CLI process uses: both keep the ambient ladder as their + // documented, legitimate posture. + // + // Set this when every caller through beads.OpenBestAvailableWith is a + // multi-tenant embedder for which ambient, process-global credential + // state can never safely stand in for one workspace's own — see + // httpclient.ResolveCredential and backends.OpenOptions.Credential's own + // doc comment. + RequireCredential bool +} + +// Register adds the http store to the backend registry under the name "http" +// AND installs the transport it dials with, so a workspace whose +// metadata.json selects the backend opens instead of failing with +// httpclient.ErrNoTransport. +// +// The EMBEDDER calls this explicitly at process start, on the same terms as +// cmd/bd's own wiring: registration is a property of the distribution being +// built, not of the import graph. A plain `go build` of an OSS binary that +// imports this package only transitively, and never calls Register, gains no +// selectable "http" backend — metadata.json naming it still hard-fails with +// the registry's UnknownBackendError, exactly as before this package existed. +// +// Register panics on a duplicate registration, like any process-start wiring +// error, so it must have exactly one production call site. Call it once, +// before any concurrent store access. +func Register(opts Options) { + base := httpclient.DialOptions{UserAgent: opts.UserAgent, HTTPClient: opts.HTTPClient} + backends.Register(httpclient.Backend, backends.Backend{ + Open: httpclient.NewFromConfig, + OpenReadOnly: httpclient.NewReadOnlyFromConfig, + WorkspaceIsBeadsDir: true, + Remote: true, + OpenWith: func(ctx context.Context, beadsDir string, bopts backends.OpenOptions) (backend.DoltStorage, error) { + return httpclient.OpenWith(ctx, beadsDir, bopts, base, opts.RequireCredential) + }, + }) + httpclient.RegisterDefaultDialer(base) +} + +// Open dials target and returns a store for it, with no workspace on disk. +// +// It is the door for an embedder that already knows its server — from its own +// configuration, or minted per tenant — and would otherwise have to write a +// .beads directory for the sole benefit of a registry lookup. Registration is +// not required and does not affect it: Open dials fresh from the Options it is +// handed, using the same built-in bearer ladder Open/OpenReadOnly use (see +// Options.RequireCredential's doc for why a per-tenant credential belongs on +// the registry's OpenWith seam instead, via beads.OpenBestAvailableWith). +// +// The store carries no local metadata file, which is benign: the two +// per-user keys it would hold read as unset and write nowhere. +func Open(ctx context.Context, target Target, opts Options) (backend.DoltStorage, error) { + dialOpts := httpclient.DialOptionsForTarget(target, httpclient.DialOptions{UserAgent: opts.UserAgent, HTTPClient: opts.HTTPClient}) + creds := httpclient.NewBearerProvider(target.BaseURL) + conn, err := httpclient.DialWith(target, creds, dialOpts) + if err != nil { + return nil, err + } + return httpclient.New(target, conn, nil), nil +} + +// Handshake dials target once and returns its startup snapshot, applying the +// two gates a store applies: api_version equality and, when the target pins +// one, project identity. +// +// It is the pre-connect probe: `bd connect` verifies a server before it +// writes anything, and it has to do that without opening a store, because the +// workspace it is about to describe may not select this backend yet. A +// mismatch comes back as *ProjectMismatchError, which names both ids plus the +// server's own database and repo root. +// +// It dials with the same credentials Open would, so a probe can never verify +// a server the store then cannot reach. +func Handshake(ctx context.Context, target Target, opts Options) (*ServerSnapshot, error) { + dialOpts := httpclient.DialOptionsForTarget(target, httpclient.DialOptions{UserAgent: opts.UserAgent, HTTPClient: opts.HTTPClient}) + // httpclient.Handshake dials via httpclient.Dial, which binds the same + // built-in bearer ladder (httpclient.NewBearerProvider) Open uses above — + // "the same credentials Open would use" holds without restating the dial + // here. + body, err := httpclient.Handshake(ctx, target, dialOpts) + if err != nil { + return nil, err + } + return newServerSnapshot(body), nil +} + +// ServerSnapshot is what a server says about itself at handshake, curated for +// this door. +// +// It is a struct of its own rather than an alias of the wire document, and +// the omissions are the point. The wire type is GENERATED from the OpenAPI +// spec, so aliasing it would make a codegen bump — a field renamed, a member +// added — a breaking change to a published Go API, decided by a document +// nobody edits with that in mind. And the document carries facts an embedder +// has no business depending on: the server's own filesystem paths, its +// storage mode, its logical database name, the CLI's JSON schema version. +// Those are the operator's, and several are host paths a multi-tenant caller +// should not be holding at all. +// +// What remains is what a client actually decides with: Capabilities is the +// field it decides with (an operation is available because the server +// advertises its token, never because a version string looked new enough), +// and WireRevision is the revision the handshake already checked against +// this client's own compiled one before returning a snapshot at all. +type ServerSnapshot struct { + // APIVersion is the path major the server serves. The handshake already + // refused anything this client cannot address, so on a returned snapshot + // this is confirmation rather than a branch. + APIVersion string + // BdVersion is the release of the serving binary. Diagnostic and + // human-facing: branch on Capabilities, not on this. + BdVersion string + // WireRevision is the server's own wire_revision counter (see + // ContextResponse.WireRevision). The wire handshake itself refuses a + // server this client build predates (a wire_revision or + // min_client_wire_revision above wire.ClientWireRevision) before any + // snapshot is returned, so on a returned snapshot this is exposed for + // diagnostics, not a branch. + WireRevision int + // ProjectID is the workspace identity the server owns. When the Target + // pinned one, the handshake has already proved they match. + ProjectID string + // Capabilities are the operation tokens the server implements. The list + // grows additively and an operation never appears unless it is fully + // implemented, which is what makes membership the right question. + Capabilities []string +} + +// newServerSnapshot maps the wire document onto the curated one. +// +// The capability slice is copied because the document it comes from is a +// pointer INTO the wire client's cached handshake, which every later dispatch +// reads to decide what the server can serve, and whose documented contract is +// that the caller treats it as read-only. +func newServerSnapshot(body *apigen.ContextResponse) *ServerSnapshot { + if body == nil { + return nil + } + return &ServerSnapshot{ + APIVersion: body.ApiVersion, + BdVersion: body.BdVersion, + WireRevision: body.WireRevision, + ProjectID: body.ProjectId, + Capabilities: append([]string(nil), body.Capabilities...), + } +} diff --git a/backend/http/open_test.go b/backend/http/open_test.go new file mode 100644 index 000000000..994124714 --- /dev/null +++ b/backend/http/open_test.go @@ -0,0 +1,235 @@ +package bdhttp_test + +// Adapted from bd-enterprise backend/http/open_test.go under the MIT +// contribution: TestOpenNeedsNoWorkspaceOnDisk, the ServerSnapshot field-set +// drift guard, TestHandshakeReportsAWrongServer, and the activation sidecar +// round trip carry the same reasoning. Dropped or reshaped for OSS: the +// Credentials-hook tests (OSS Open/Handshake dial through the built-in bearer +// ladder only — see Options.RequireCredential's doc — there is no per-call +// hook to assert on here), ServerInfo / lazy ServerContext (the OSS Store +// does not expose that interface), and ServerSnapshot's field set itself, +// which follows DESIGN.txt exactly (WireRevision in place of the ORM-backed +// field bd-enterprise used). No bd-enterprise-specific wording carries over. + +import ( + "context" + "errors" + "net/http" + "net/url" + "path/filepath" + "reflect" + "testing" + + bdhttp "github.com/steveyegge/beads/backend/http" +) + +// TestOpenNeedsNoWorkspaceOnDisk is the programmatic door's reason to exist. +// An embedder that already knows the server — it read the URL from its own +// config, or minted a per-tenant one — has no .beads directory to point the +// registry at, and creating one just to open a store would be a file written +// for the benefit of a lookup. +func TestOpenNeedsNoWorkspaceOnDisk(t *testing.T) { + hermeticEnv(t) + t.Setenv(bdhttp.TokenEnv, "127.0.0.1=direct-token") + server := newRecordingServer(t) + + store, err := bdhttp.Open(context.Background(), server.target(t), bdhttp.Options{HTTPClient: server.Client()}) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + got, err := store.GetMetadata(context.Background(), "_project_id") + if err != nil { + t.Fatalf("GetMetadata: %v", err) + } + if got != projectID { + t.Errorf("project id = %q, want %q", got, projectID) + } + server.wantAuthorization(t, "Bearer direct-token") +} + +// TestOpenWithoutRegistrationStillDials pins the independence of the two +// doors: Open dials from the Options it is handed, with no dependence on +// Register ever having run. +func TestOpenWithoutRegistrationStillDials(t *testing.T) { + hermeticEnv(t) + t.Setenv(bdhttp.TokenEnv, "127.0.0.1=unregistered") + server := newRecordingServer(t) + + store, err := bdhttp.Open(context.Background(), server.target(t), bdhttp.Options{HTTPClient: server.Client()}) + if err != nil { + t.Fatalf("Open without Register: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if _, err := store.GetMetadata(context.Background(), "_project_id"); err != nil { + t.Fatalf("GetMetadata: %v", err) + } + server.wantAuthorization(t, "Bearer unregistered") +} + +// TestHandshakeVerifiesBeforeAnythingIsWritten is the connect path's probe. A +// caller about to record a server has to be able to check it first, and doing +// that by opening a store would mean describing a workspace before knowing +// whether the server behind it is the right one. +func TestHandshakeVerifiesBeforeAnythingIsWritten(t *testing.T) { + hermeticEnv(t) + server := newRecordingServer(t) + + snapshot, err := bdhttp.Handshake(context.Background(), server.target(t), bdhttp.Options{HTTPClient: server.Client()}) + if err != nil { + t.Fatalf("Handshake: %v", err) + } + if snapshot.ProjectID != projectID { + t.Errorf("project id = %q, want %q", snapshot.ProjectID, projectID) + } + if snapshot.BdVersion != "9.9.9" { + t.Errorf("bd_version = %q, want the server's own", snapshot.BdVersion) + } + if snapshot.APIVersion != "v0" { + t.Errorf("api_version = %q, want the major this client speaks", snapshot.APIVersion) + } + if snapshot.WireRevision != 1 { + t.Errorf("wire_revision = %d, want the server's own", snapshot.WireRevision) + } + if len(snapshot.Capabilities) != 2 || snapshot.Capabilities[0] != "issues.get" { + t.Errorf("capabilities = %v, want the server's advertised list", snapshot.Capabilities) + } +} + +// TestServerSnapshotExportsNoHostFacts is the drift guard on the curated +// snapshot, and it is a guard rather than a comment because the thing it +// prevents arrives by accident. +// +// The wire document is GENERATED from the OpenAPI spec. Aliasing it, or +// widening this struct to match it, would put a codegen bump — a renamed +// field, a new member — on a published Go API, and would hand every embedder +// the server's own filesystem paths. A multi-tenant caller holding another +// tenant's repo root is not a field it chose to read; it is a field it could +// not avoid. +func TestServerSnapshotExportsNoHostFacts(t *testing.T) { + want := map[string]bool{ + "APIVersion": true, + "BdVersion": true, + "WireRevision": true, + "ProjectID": true, + "Capabilities": true, + } + typ := reflect.TypeOf(bdhttp.ServerSnapshot{}) + if typ.PkgPath() != "github.com/steveyegge/beads/backend/http" { + t.Fatalf("ServerSnapshot is declared in %q: it must be this package's own struct, not an alias of the generated wire type", typ.PkgPath()) + } + for i := 0; i < typ.NumField(); i++ { + name := typ.Field(i).Name + if !want[name] { + t.Errorf("ServerSnapshot exports %q, which is not on the curated set: server host paths, storage mode, logical database and the CLI schema version stay behind this door", name) + continue + } + delete(want, name) + } + for name := range want { + t.Errorf("ServerSnapshot no longer carries %q", name) + } +} + +// TestHandshakeReportsAWrongServer pins the gate that makes the probe worth +// running: the pinned identity is checked, and the refusal is typed, so a +// caller can tell "that is the wrong server" from "that server is down" +// without matching text. +func TestHandshakeReportsAWrongServer(t *testing.T) { + hermeticEnv(t) + server := newRecordingServer(t) + target := server.target(t) + target.ExpectProjectID = "proj-somewhere-else" + + _, err := bdhttp.Handshake(context.Background(), target, bdhttp.Options{HTTPClient: server.Client()}) + if !errors.Is(err, bdhttp.ErrProjectMismatch) { + t.Fatalf("Handshake against a wrong server = %v, want ErrProjectMismatch", err) + } + var mismatch *bdhttp.ProjectMismatchError + if !errors.As(err, &mismatch) { + t.Fatalf("the refusal is not a *ProjectMismatchError: %v", err) + } +} + +// TestHandshakeUsesTheSameLadderOpenWouldUse keeps the two doors on one +// credential path: a probe that authorized differently from the store it +// precedes would verify a server the store then cannot reach. +func TestHandshakeUsesTheSameLadderOpenWouldUse(t *testing.T) { + hermeticEnv(t) + t.Setenv(bdhttp.TokenEnv, "127.0.0.1=probe-token") + server := newRecordingServer(t) + + if _, err := bdhttp.Handshake(context.Background(), server.target(t), bdhttp.Options{HTTPClient: server.Client()}); err != nil { + t.Fatalf("Handshake: %v", err) + } + server.wantAuthorization(t, "Bearer probe-token") +} + +// TestTheActivationSidecarRoundTrips covers the workspace half of the +// surface: an embedder that writes a connected workspace for a later `bd` +// invocation, or reads the one `bd connect` already wrote, goes through +// these. +func TestTheActivationSidecarRoundTrips(t *testing.T) { + beadsDir := t.TempDir() + + if _, err := bdhttp.LoadTarget(beadsDir); !errors.Is(err, bdhttp.ErrNotConnected) { + t.Fatalf("LoadTarget on an unconnected workspace = %v, want ErrNotConnected", err) + } + if removed, err := bdhttp.RemoveTarget(beadsDir); err != nil || removed { + t.Fatalf("RemoveTarget on an unconnected workspace = (%v, %v), want (false, nil)", removed, err) + } + + base, err := url.Parse("https://beads.example.com") + if err != nil { + t.Fatal(err) + } + want := bdhttp.Target{BaseURL: base, ExpectProjectID: projectID} + if err := bdhttp.SaveTarget(beadsDir, want); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + if got := bdhttp.TargetPath(beadsDir); got != filepath.Join(beadsDir, bdhttp.TargetFileName) { + t.Errorf("TargetPath = %q, want the sidecar beside metadata.json", got) + } + + got, err := bdhttp.LoadTarget(beadsDir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.String() != want.String() || got.ExpectProjectID != want.ExpectProjectID { + t.Errorf("round trip = %+v, want %+v", got, want) + } + + removed, err := bdhttp.RemoveTarget(beadsDir) + if err != nil { + t.Fatalf("RemoveTarget: %v", err) + } + if !removed { + t.Error("RemoveTarget did not report the sidecar it deleted") + } + if _, err := bdhttp.LoadTarget(beadsDir); !errors.Is(err, bdhttp.ErrNotConnected) { + t.Errorf("LoadTarget after RemoveTarget = %v, want ErrNotConnected", err) + } +} + +// TestABearerProviderIsBuildableFromThePublicSurface proves the default +// ladder is nameable, which is what lets a caller wrap or inspect it without +// reimplementing it. +func TestABearerProviderIsBuildableFromThePublicSurface(t *testing.T) { + hermeticEnv(t) + t.Setenv(bdhttp.TokenEnv, "127.0.0.1=ladder-token") + server := newRecordingServer(t) + + var provider bdhttp.CredentialProvider = bdhttp.NewBearerProvider(server.target(t).BaseURL) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, server.URL, nil) + if err != nil { + t.Fatal(err) + } + if err := provider.Authorize(context.Background(), req); err != nil { + t.Fatalf("Authorize: %v", err) + } + if got := req.Header.Get("Authorization"); got != "Bearer ladder-token" { + t.Errorf("Authorization = %q, want the ladder's token", got) + } +} diff --git a/backend/http/register_test.go b/backend/http/register_test.go new file mode 100644 index 000000000..4ed4b1c3c --- /dev/null +++ b/backend/http/register_test.go @@ -0,0 +1,318 @@ +package bdhttp_test + +// Test helpers in this file are adapted from bd-enterprise +// backend/http/register_test.go under the MIT contribution: the registry +// double, the recording server/provider, and hermeticEnv's reasoning for +// taking the process environment out of credential decisions carry over +// directly. The Credentials-hook tests do not: OSS threads a per-open +// credential through backends.OpenOptions.Credential / httpclient.OpenWith +// instead of a bdhttp.Options.Credentials hook (no such hook exists on this +// path; see design D-OpenWith), so those tests are written fresh for this +// seam. + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/steveyegge/beads/backend" + bdhttp "github.com/steveyegge/beads/backend/http" + "github.com/steveyegge/beads/internal/storage/backends" +) + +// The registry key is spelled literally, the way a workspace's metadata.json +// carries it: a test that read it from the code under test would stay green +// through a rename that orphans every existing connected workspace. +const backendName = "http" + +// projectID is the workspace identity the stand-in server owns. Every target +// below pins it, so the handshake's identity gate is armed rather than +// skipped. +const projectID = "proj-embedder" + +// TestRegisterPutsHTTPInTheRegistryAnEmbedderSees drives the reason this +// package exists: code holding nothing but the public surface can get "http" +// into the registry that store dispatch reads. +func TestRegisterPutsHTTPInTheRegistryAnEmbedderSees(t *testing.T) { + hermeticEnv(t) + if backend.Registered(backendName) { + t.Fatalf("%q was already registered before Register was called: linking this package must not register", backendName) + } + + registerForTest(t, bdhttp.Options{}) + + registered, ok := backend.Lookup(backendName) + if !ok { + t.Fatalf("Lookup(%q) found nothing after Register", backendName) + } + if registered.Open == nil { + t.Error("registered http backend has no read-write Open") + } + if registered.OpenReadOnly == nil { + t.Error("registered http backend has no read-only Open") + } + if !registered.WorkspaceIsBeadsDir { + t.Error("an http workspace is the .beads directory alone; there is no local database to discover") + } + if !registered.Remote { + t.Error("the http backend is a pure network client with no local database; Remote must be set") + } + if registered.OpenWith == nil { + t.Error("registered http backend has no per-open OpenWith seam") + } + if !backend.WorkspaceIsBeadsDir(backendName) { + t.Error("backend.WorkspaceIsBeadsDir disagrees with the registration") + } + if !backend.IsRemote(backendName) { + t.Error("backend.IsRemote disagrees with the registration") + } +} + +// TestRegisterTwicePanics pins the decision not to make this idempotent. A +// second call is a double-wired process start, and the registry's panic is +// the only place that ever says so. +func TestRegisterTwicePanics(t *testing.T) { + hermeticEnv(t) + registerForTest(t, bdhttp.Options{}) + + if !panics(func() { bdhttp.Register(bdhttp.Options{}) }) { + t.Fatal("a second Register returned instead of panicking: duplicate process-start wiring must not be absorbed") + } + if !backend.Registered(backendName) { + t.Fatalf("the refused duplicate took %q out of the registry", backendName) + } +} + +// TestRegisterInstallsTheTransport is the whole reason Register takes +// Options: registration alone leaves the store's package-level dialer nil, +// and every open then fails with httpclient.ErrNoTransport — a two-step an +// embedder cannot complete, because the second step is an internal function. +func TestRegisterInstallsTheTransport(t *testing.T) { + hermeticEnv(t) + server := newRecordingServer(t) + registerForTest(t, bdhttp.Options{HTTPClient: server.Client()}) + + store := openWorkspace(t, connectedWorkspace(t, server)) + got, err := store.GetMetadata(context.Background(), "_project_id") + if err != nil { + t.Fatalf("GetMetadata over the registered backend: %v", err) + } + if got != projectID { + t.Errorf("project id = %q, want %q: the store never reached the server", got, projectID) + } +} + +// TestOpenWithHonorsAProvidedCredential pins the per-tenant seam Register +// installs: a caller that supplies backends.OpenOptions.Credential (wrapped +// in bdhttp.ProvidedCredential) reaches the server with THAT credential, not +// the ambient ladder, even though the registered backend was built with no +// credential of its own. +func TestOpenWithHonorsAProvidedCredential(t *testing.T) { + hermeticEnv(t) + server := newRecordingServer(t) + registerForTest(t, bdhttp.Options{HTTPClient: server.Client()}) + beadsDir := connectedWorkspace(t, server) + + registered, ok := backend.Lookup(backendName) + if !ok { + t.Fatal("the http backend is not registered") + } + provider := &recordingProvider{token: "per-tenant-token"} + store, err := registered.OpenWith(context.Background(), beadsDir, backends.OpenOptions{ + Credential: bdhttp.ProvidedCredential{Provider: provider}, + }) + if err != nil { + t.Fatalf("OpenWith: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if _, err := store.GetMetadata(context.Background(), "_project_id"); err != nil { + t.Fatalf("GetMetadata: %v", err) + } + server.wantAuthorization(t, "Bearer per-tenant-token") +} + +// TestOpenWithFallsBackToTheAmbientLadderByDefault pins the single-tenant +// default: a registered open with no Credential still authorizes, through +// the same built-in bearer ladder Open/OpenReadOnly use — RequireCredential +// is what opts OUT of this, not the zero value. +func TestOpenWithFallsBackToTheAmbientLadderByDefault(t *testing.T) { + hermeticEnv(t) + t.Setenv(bdhttp.TokenEnv, "127.0.0.1=ambient-token") + server := newRecordingServer(t) + registerForTest(t, bdhttp.Options{HTTPClient: server.Client()}) + beadsDir := connectedWorkspace(t, server) + + store := openWorkspace(t, beadsDir) + if _, err := store.GetMetadata(context.Background(), "_project_id"); err != nil { + t.Fatalf("GetMetadata: %v", err) + } + server.wantAuthorization(t, "Bearer ambient-token") +} + +// TestRequireCredentialRefusesAnOpenWithNone is the fail-closed half: a +// multi-tenant embedder that sets RequireCredential must never have a +// workspace silently fall back to whatever the ambient process environment +// happens to hold, because that environment belongs to no one tenant. +func TestRequireCredentialRefusesAnOpenWithNone(t *testing.T) { + hermeticEnv(t) + server := newRecordingServer(t) + registerForTest(t, bdhttp.Options{HTTPClient: server.Client(), RequireCredential: true}) + beadsDir := connectedWorkspace(t, server) + + registered, ok := backend.Lookup(backendName) + if !ok { + t.Fatal("the http backend is not registered") + } + _, err := registered.OpenWith(context.Background(), beadsDir, backends.OpenOptions{}) + if err == nil { + t.Fatal("OpenWith with RequireCredential and no Credential succeeded; want a refusal") + } + server.wantNoRequests(t) +} + +func panics(fn func()) (panicked bool) { + defer func() { panicked = recover() != nil }() + fn() + return false +} + +// registerForTest wires the backend for one test and gives the process-global +// registry back afterwards, the isolation seam backend.Deregister exists for. +func registerForTest(t *testing.T, opts bdhttp.Options) { + t.Helper() + bdhttp.Register(opts) + t.Cleanup(func() { backend.Deregister(backendName) }) +} + +// connectedWorkspace is a .beads directory attached to server, as `bd +// connect` would leave it. +func connectedWorkspace(t *testing.T, server *recordingServer) string { + t.Helper() + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create workspace: %v", err) + } + if err := bdhttp.SaveTarget(beadsDir, server.target(t)); err != nil { + t.Fatalf("save the activation sidecar: %v", err) + } + return beadsDir +} + +// openWorkspace opens beadsDir through the REGISTERED backend rather than +// through this package's own Open, so what it exercises is the dialer +// Register installed. +func openWorkspace(t *testing.T, beadsDir string) backend.DoltStorage { + t.Helper() + registered, ok := backend.Lookup(backendName) + if !ok { + t.Fatal("the http backend is not registered") + } + store, err := registered.Open(context.Background(), beadsDir) + if err != nil { + t.Fatalf("open the connected workspace: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + return store +} + +// hermeticEnv takes the process environment out of the credential decision. +// Every rung of the built-in ladder is read from the environment at dial +// time, so a developer machine that exports one would otherwise change which +// source a test exercises. +func hermeticEnv(t *testing.T) { + t.Helper() + t.Setenv(bdhttp.TokenEnv, "") + t.Setenv(bdhttp.TokenCommandEnv, "") +} + +// recordingProvider is a CredentialProvider that signs with a fixed token. +type recordingProvider struct { + token string + + mu sync.Mutex + calls int +} + +func (p *recordingProvider) Authorize(_ context.Context, req *http.Request) error { + p.mu.Lock() + p.calls++ + p.mu.Unlock() + req.Header.Set("Authorization", "Bearer "+p.token) + return nil +} + +func (p *recordingProvider) Refresh(context.Context) (bool, error) { return false, nil } + +// recordingServer is a bd serve stand-in that answers the v0 handshake and +// records what each request carried in its Authorization header. +type recordingServer struct { + *httptest.Server + + mu sync.Mutex + auths []string +} + +func newRecordingServer(t *testing.T) *recordingServer { + t.Helper() + s := &recordingServer{} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.auths = append(s.auths, r.Header.Get("Authorization")) + s.mu.Unlock() + if r.URL.Path != "/v0/beads/context" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"api_version":"v0","backend":"dolt","bd_version":"9.9.9",` + + `"beads_dir":"/srv/.beads","capabilities":["issues.get","issues.list"],` + + `"database":"beads","dolt_mode":"server","project_id":"` + projectID + `","repo_root":"/srv",` + + `"schema_version":1,"wire_revision":1,"min_client_wire_revision":1}`)) + })) + t.Cleanup(s.Server.Close) + return s +} + +func (s *recordingServer) target(t *testing.T) bdhttp.Target { + t.Helper() + base, err := url.Parse(s.URL) + if err != nil { + t.Fatalf("parse the test server URL: %v", err) + } + return bdhttp.Target{BaseURL: base, ExpectProjectID: projectID} +} + +// wantAuthorization asserts that every request the server saw carried want. +// It is every request rather than any request because a credential that +// reaches only some of them is the same defect as one that reaches none. +func (s *recordingServer) wantAuthorization(t *testing.T, want string) { + t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + if len(s.auths) == 0 { + t.Fatal("the server saw no requests at all") + } + for i, got := range s.auths { + if got != want { + t.Errorf("request %d carried Authorization %q, want %q", i, got, want) + } + } +} + +// wantNoRequests is the fail-closed half: a refusal that still dialed would +// have sent whatever credential it fell back to, which is the outcome being +// refused. +func (s *recordingServer) wantNoRequests(t *testing.T) { + t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + if len(s.auths) != 0 { + t.Errorf("the server saw %d requests, want none: the dial was supposed to fail closed", len(s.auths)) + } +} diff --git a/backend/http/target.go b/backend/http/target.go new file mode 100644 index 000000000..ab6ee2917 --- /dev/null +++ b/backend/http/target.go @@ -0,0 +1,41 @@ +package bdhttp + +import "github.com/steveyegge/beads/internal/httpclient" + +// Target is the resolved endpoint for one workspace: the server's mount root +// and, optionally, the workspace identity pinned when it was connected. A +// pinned id is compared against the server's own at every handshake, which is +// what turns "some server answered" into "the server that owns this workspace +// answered". +type Target = httpclient.Target + +// TargetFileName is the per-user, untracked activation sidecar that names the +// server a workspace is attached to. metadata.json is git-tracked by design +// and selects the backend; the URL and the identity pin are per-user state +// and live beside it. +const TargetFileName = httpclient.TargetFileName + +// ErrNotConnected reports a workspace that selects this backend but has no +// activation sidecar, so there is no server to dial. It is what an Open +// through the registry returns for a workspace nobody ran `bd connect` in. +var ErrNotConnected = httpclient.ErrNotConnected + +// TargetPath is the sidecar's location for a workspace, given its .beads +// directory. +func TargetPath(beadsDir string) string { return httpclient.TargetPath(beadsDir) } + +// LoadTarget reads the activation sidecar. A missing one is ErrNotConnected +// rather than a zero Target: an http workspace with no target cannot be +// opened at all. +func LoadTarget(beadsDir string) (Target, error) { return httpclient.LoadTarget(beadsDir) } + +// SaveTarget writes the sidecar 0600, beside metadata.json. It is the encoder +// `bd connect` uses; the connect UX around it — gitignore coverage, identity +// verification, conversion consent — is the CLI's, so an embedder that writes +// a workspace this way owns those decisions itself. +func SaveTarget(beadsDir string, t Target) error { return httpclient.SaveTarget(beadsDir, t) } + +// RemoveTarget deletes the activation sidecar, reporting whether one was +// there. Detaching is a per-user act, so it never touches the tracked +// metadata.json that selects the backend. +func RemoveTarget(beadsDir string) (bool, error) { return httpclient.RemoveTarget(beadsDir) } diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index eb8aa6cd2..b6cca03ee 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -62,6 +62,7 @@ go_library( "config_show.go", "config_side_effects.go", "conflicts.go", + "connect.go", "context.go", "context_cmd.go", "context_proxied_server.go", @@ -157,6 +158,7 @@ go_library( "history.go", "history_proxied_server.go", "hooks.go", + "http_backend.go", "human.go", "human_proxied_server.go", "if_revision.go", @@ -366,6 +368,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//:beads", + "//backend/http", "//cmd/bd/doctor", "//cmd/bd/doctor/fix", "//cmd/bd/setup", @@ -392,6 +395,7 @@ go_library( "//internal/gitlab", "//internal/hooks", "//internal/httpapi", + "//internal/httpclient", "//internal/jira", "//internal/linear", "//internal/lockfile", @@ -493,6 +497,7 @@ go_binary( embed = [":bd_lib"], race = "off", visibility = [ + "//backend/http:__pkg__", "//cmd/bd:__subpackages__", "//internal/beads:__pkg__", "//internal/storage/uow:__pkg__", diff --git a/cmd/bd/capability_registry.go b/cmd/bd/capability_registry.go index de387c97e..1ac36eca9 100644 --- a/cmd/bd/capability_registry.go +++ b/cmd/bd/capability_registry.go @@ -511,7 +511,7 @@ var proxyPermittedPaths = []string{ // workspace lifecycle and configuration "bootstrap", "config apply", "config drift", "config get", "config list", "config set", - "config set-many", "config show", "config unset", "config validate", "context", + "config set-many", "config show", "config unset", "config validate", "connect", "context", "init", "init-safety", "onboard", "preflight", "quickstart", "schema", "setup", "where", "version", "worktree create", "worktree info", "worktree list", "worktree remove", diff --git a/cmd/bd/connect.go b/cmd/bd/connect.go new file mode 100644 index 000000000..29f369998 --- /dev/null +++ b/cmd/bd/connect.go @@ -0,0 +1,324 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + + "github.com/spf13/cobra" + bdhttp "github.com/steveyegge/beads/backend/http" + "github.com/steveyegge/beads/cmd/bd/doctor" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/httpclient" + "github.com/steveyegge/beads/internal/metrics" +) + +// Written fresh for OSS beads S6 (no bd-enterprise source copied): richer +// connect flows (interactive login, server discovery, storage-profile +// prompts) are explicitly out of scope for this lift (see DESIGN.txt sec +// 3.1). This is the minimal OSS connect DESIGN.txt asks for: verify a +// server, then record it. + +var ( + connectExpectProjectID string + connectCAFile string + connectForce bool + connectAllowPlaintext bool + connectClear bool +) + +var connectCmd = &cobra.Command{ + Use: "connect [url]", + GroupID: "setup", + Short: "Attach this workspace to a remote bd serve over HTTP", + Long: `Attach this workspace to a remote bd serve over HTTP. + +bd connect verifies the server first (a handshake: api_version, wire +revision, and — when --expect-project-id is given — workspace identity) and +writes nothing until that succeeds. On success it writes the per-user +activation sidecar (.beads/http_target.json, never git-tracked) and sets this +workspace's metadata.json to backend "http". + +A credential, if the server requires one, is never read from a flag or +recorded on disk by this command. It comes from the same ladder every http +request does: the BEADS_HTTP_TOKEN environment variable, then +BEADS_HTTP_TOKEN_COMMAND (a helper that prints a token), then the credentials +file's [host:port] section, then no credential at all — which the tip OSS +server's loopback-trust posture answers legitimately, not as a failure. Both +variables name the server they are for, host[:port]=value (for example +BEADS_HTTP_TOKEN=bd.example.com=); a bare value is refused, since it +would be sent to whatever server the workspace's http_target.json names. + +--allow-plaintext IS recorded on disk, in the sidecar alongside the server +url: once granted here, a credential may cross this one server in the clear +on every later command too, not only this one, without re-passing the flag +or setting BEADS_HTTP_ALLOW_INSECURE=1 yourself. A later connect to a +DIFFERENT url starts that grant over; it is never carried forward to a +server it was not given for. + +Examples: + bd connect http://127.0.0.1:8080 # loopback, no TLS needed + bd connect https://bd.example.com --expect-project-id p1 # pin workspace identity + bd connect https://bd.example.com --ca-file ./ca.pem # trust only this CA + bd connect --clear # forget the server, restoring the previous backend +`, + Args: cobra.MaximumNArgs(1), + SilenceUsage: true, + SilenceErrors: true, + // connect is how an http workspace gets its FIRST local state: there is + // no local database to find yet (none ever exists for this backend), so + // it opts out of PersistentPreRunE's store init the same way doctor, + // schema and worktree do, via skipStoreAnnotation rather than editing + // the central noDbCommands list (see that annotation's own doc comment). + Annotations: map[string]string{skipStoreAnnotation: "1"}, + RunE: func(cmd *cobra.Command, args []string) error { + evt := metrics.NewCommandEvent("connect") + defer func() { + if c := metrics.Global(); c != nil { + c.CloseEventAndAdd(evt) + } + }() + + // connect, like init, is how a workspace gets its FIRST local state: + // there may be no .beads directory yet anywhere above the CWD, so + // this resolves the same way init's own target does (BEADS_DIR, then + // the worktree fallback, then CWD/.beads) rather than + // beads.GetRepoContextAllowingNoGit(), which answers "no .beads + // directory found" with a hard error because every OTHER caller of + // it (bd context, the proxied context route) is read-only and has + // nothing to create. + beadsDir := resolveInitBeadsDir() + if beadsDir == "" { + return HandleError("cannot resolve a workspace directory for connect (set BEADS_DIR, or run from inside a writable directory)") + } + + if connectClear { + if len(args) != 0 { + return HandleError("connect --clear takes no url argument") + } + // Read the sidecar's PreviousBackend BEFORE removing it: it is + // the backend metadata.json selected right before this + // workspace connected (bee-ghosttrack CHANGES_REQUESTED on + // #7288, should-fix 1). A missing sidecar (ErrNotConnected) or + // any other read failure just means there is nothing to + // restore, matching the "nothing was configured" case below. + priorTarget, loadErr := httpclient.LoadTarget(beadsDir) + removed, err := httpclient.RemoveTarget(beadsDir) + if err != nil { + return HandleError("clearing %s: %v", httpclient.TargetFileName, err) + } + restoredBackend := "" + if removed && loadErr == nil && priorTarget.PreviousBackend != "" { + if cfg, cfgErr := configfile.Load(beadsDir); cfgErr == nil && cfg != nil && cfg.GetBackend() == httpclient.Backend { + cfg.Backend = priorTarget.PreviousBackend + if saveErr := cfg.Save(beadsDir); saveErr == nil { + restoredBackend = priorTarget.PreviousBackend + } + // A save failure is not fatal to --clear: the sidecar is + // already gone (or never existed), which is the part + // that actually detaches the workspace from the + // server; metadata.json just keeps saying "http" and + // the manual fallback message below still applies. + } + } + if jsonOutput { + return outputJSON(map[string]any{"cleared": removed, "restored_backend": restoredBackend}) + } + switch { + case restoredBackend != "": + fmt.Printf("Removed %s and restored this workspace's backend selection in metadata.json to %q.\n", httpclient.TargetFileName, restoredBackend) + case removed: + // Not `bd config set backend ...`: that writes the database's + // own config table, never metadata.json, and must first open + // the very store this workspace can no longer reach. + fmt.Printf("Removed %s. This workspace's backend selection in metadata.json is unchanged; run `bd connect ` to reconnect, or set \"backend\" in %s to pick a different backend.\n", httpclient.TargetFileName, filepath.Join(beadsDir, configfile.ConfigFileName)) + default: + fmt.Println("No http target was configured; nothing to clear.") + } + return nil + } + + if len(args) != 1 { + return HandleError("connect requires a argument (or --clear)") + } + rawURL := args[0] + + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Host == "" { + // A parse failure has no *url.URL to redact, so there is nothing + // safer to print than the raw argument; a parse that succeeded + // but found no host (e.g. "http://user:pass@/path") does have + // one, and .Redacted() strips its userinfo before this message + // repeats it. + if err != nil { + return HandleError("invalid url %q", rawURL) + } + return HandleError("invalid url %q", parsed.Redacted()) + } + // redactedURL is what every print/wrap of the connect target uses from + // here on: rawURL is the user's literal argument, which — unlike + // parsed.Redacted() — still carries a userinfo password verbatim + // (http://user:pass@host/...) if one was pasted into the URL. Nothing + // past this point should format rawURL again. + redactedURL := parsed.Redacted() + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return HandleError("url %q must use http or https, got %q", redactedURL, parsed.Scheme) + } + // L1 (host safety, functional requirement 1): a bearer credential sent + // in the clear to anything but loopback is a token leaked to every + // network hop between here and the server. Refuse by default; require + // an explicit opt-in rather than a warning nobody reads at connect + // time, since connect time is the one moment a human is looking. + if parsed.Scheme == "http" && !configfile.IsLocalHostString(parsed.Hostname()) && !connectAllowPlaintext { + return HandleError( + "refusing to connect to %s over plain http: %s is not loopback, and any bearer credential this workspace sends would cross the network unencrypted; use https://, or pass --allow-plaintext to connect anyway", + redactedURL, parsed.Hostname()) + } + + existing, err := configfile.Load(beadsDir) + if err != nil { + return HandleError("reading %s: %v", configfile.ConfigFileName, err) + } + if existing != nil { + if b := existing.GetBackend(); b != "" && b != httpclient.Backend && !connectForce { + return HandleError("this workspace already selects backend %q; pass --force to switch it to %q (existing local data, if any, is left in place but will no longer be used)", b, httpclient.Backend) + } + } + + // effectiveExpectedProjectID carries the PRIOR sidecar's pin forward + // into this run, so a bare re-connect (no --expect-project-id) to a + // DIFFERENT server still gets caught: without this, every connect + // built a brand-new Target starting from connectExpectProjectID alone + // (almost always "" on a re-run), which the "nothing was pinned yet" + // branch below would then adopt from whatever server just answered — + // silently re-pinning the workspace to a different project with no + // refusal at all. --force (regardless of --expect-project-id) drops + // the prior pin entirely and lets the new server's answer win, which + // is the explicit "I mean it" escape hatch this check otherwise has + // none of. + effectiveExpectedProjectID := connectExpectProjectID + if !connectForce { + priorTarget, loadErr := httpclient.LoadTarget(beadsDir) + if loadErr != nil && !errors.Is(loadErr, httpclient.ErrNotConnected) { + return HandleError("reading existing %s: %v", httpclient.TargetFileName, loadErr) + } + if loadErr == nil && priorTarget.ExpectProjectID != "" { + switch { + case effectiveExpectedProjectID == "": + effectiveExpectedProjectID = priorTarget.ExpectProjectID + case effectiveExpectedProjectID != priorTarget.ExpectProjectID: + return HandleError( + "this workspace is already connected to project %q; pass --force to re-pin it to %q, or pass --expect-project-id %q to confirm the existing pin", + priorTarget.ExpectProjectID, effectiveExpectedProjectID, priorTarget.ExpectProjectID) + } + } + } + + var caFile string + dialOpts := httpclient.DialOptions{ + UserAgent: "bd/" + Version + " " + httpclient.WireUserAgentSuffix, + // The L1 refusal above already gated this exact combination + // (plain http, non-loopback) on --allow-plaintext; carrying the + // flag through here keeps this command's own Handshake probe + // (which dials with the same ambient bearer ladder a credential + // might come from) from re-refusing what the flag already + // permitted. See DialOptions.AllowInsecureCredential. + AllowInsecureCredential: connectAllowPlaintext, + } + if connectCAFile != "" { + abs, err := filepath.Abs(connectCAFile) + if err != nil { + return HandleError("resolving --ca-file %q: %v", connectCAFile, err) + } + caFile = abs + // Verify against EXACTLY this file, bypassing BEADS_HTTP_CA_FILE + // entirely: a connect run with the env var set to a different CA + // must not verify against the env's CA while writing the flag's + // unverified one to the sidecar (see DialOptionsForFile's doc). + dialOpts, err = httpclient.DialOptionsForFile(caFile, dialOpts) + if err != nil { + return HandleError("--ca-file %q: %v", connectCAFile, err) + } + } + + target := httpclient.Target{BaseURL: parsed, ExpectProjectID: effectiveExpectedProjectID, CAFile: caFile, AllowInsecureCredential: connectAllowPlaintext} + + ctx := context.Background() + snapshot, err := httpclient.Handshake(ctx, target, dialOpts) + if err != nil { + return HandleError("connecting to %s: %v", redactedURL, err) + } + if target.ExpectProjectID == "" { + // Nothing was pinned (a genuine first connect, or --force + // dropped the prior pin above): record what the server said so + // this workspace is tied to THIS project from here on. + target.ExpectProjectID = snapshot.ProjectId + } else if target.ExpectProjectID != snapshot.ProjectId { + // effectiveExpectedProjectID is either an explicit + // --expect-project-id or the prior sidecar's pin carried + // forward above; either way the server disagrees, and --force + // was not given (the only way to reach this line with a + // mismatched prior pin AND --force would have cleared + // effectiveExpectedProjectID's carry-forward, not left it set). + return HandleError("server at %s owns project %q, not the pinned %q", redactedURL, snapshot.ProjectId, target.ExpectProjectID) + } + + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + return HandleError("creating %s: %v", beadsDir, err) + } + // A fresh .beads directory (the common case: this backend has no + // local database to have created one earlier) has no .gitignore yet. + // Ensure one before writing the sidecar, the same way bd init and + // doctor --fix do, so http_target.json (which carries a bearer + // credential's host and, soon, DialOptions) is never a candidate for + // `git add .` in the first commit after connect. + if err := doctor.EnsureGitignoreForBeadsDir(beadsDir); err != nil { + return HandleError("ensuring %s/.gitignore: %v", beadsDir, err) + } + // bdhttp.Attach writes both of this backend's files: the gitignored + // identity sidecar (SaveTarget — which server, which project; see + // NewFromConfig/NewReadOnlyFromConfig in internal/httpclient/store.go, + // which read identity from the sidecar alone, never from + // metadata.json) and metadata.json's backend SELECTION alone + // (git-tracked, the same file every clone of this repo shares). It + // leaves every other existing metadata.json field untouched — in + // particular cfg.Database/cfg.ProjectID, which belong to whatever + // backend this workspace selected before, and which a future + // `bd connect --clear` switching back to that backend (dolt) still + // needs intact. + if err := bdhttp.Attach(beadsDir, target); err != nil { + return HandleError("%v", err) + } + + if jsonOutput { + return outputJSON(map[string]any{ + "url": redactedURL, + "project_id": target.ExpectProjectID, + "bd_version": snapshot.BdVersion, + "api_version": snapshot.ApiVersion, + "wire_revision": snapshot.WireRevision, + "ca_file": caFile, + "capabilities": snapshot.Capabilities, + }) + } + fmt.Printf("Connected %s to %s\n", beadsDir, redactedURL) + fmt.Printf(" project: %s\n", target.ExpectProjectID) + fmt.Printf(" bd serve: %s (api %s, wire revision %d)\n", snapshot.BdVersion, snapshot.ApiVersion, snapshot.WireRevision) + if caFile != "" { + fmt.Printf(" ca_file: %s\n", caFile) + } + return nil + }, +} + +func init() { + connectCmd.Flags().StringVar(&connectExpectProjectID, "expect-project-id", "", "pin the workspace identity the server must present at every handshake") + connectCmd.Flags().StringVar(&connectCAFile, "ca-file", "", "trust ONLY this PEM file for this server (resolved and recorded as an absolute path)") + connectCmd.Flags().BoolVar(&connectForce, "force", false, "allow switching a workspace that already selects a different backend") + connectCmd.Flags().BoolVar(&connectAllowPlaintext, "allow-plaintext", false, "allow a plain http connection to a non-loopback host; remembered in the sidecar for every later command against this server") + connectCmd.Flags().BoolVar(&connectClear, "clear", false, "remove this workspace's http activation sidecar and restore its previous backend selection in metadata.json") + rootCmd.AddCommand(connectCmd) +} diff --git a/cmd/bd/context_cmd.go b/cmd/bd/context_cmd.go index 3353f7213..fa3bdf91f 100644 --- a/cmd/bd/context_cmd.go +++ b/cmd/bd/context_cmd.go @@ -7,6 +7,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/httpclient" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/storage/domain" ) @@ -135,7 +136,19 @@ Examples: // while each held its own copy of `Backend: configfile.BackendDolt`. func applyContextBackend(snapshot *domain.ContextInfo, beadsDir string, cfg *configfile.Config) error { snapshot.SetBackendIdentity(cfg.GetBackend(), cfg.GetDoltMode(), cfg.GetDoltDatabase()) - snapshot.ProjectID = cfg.ProjectID + if cfg.GetBackend() == httpclient.Backend { + // The http backend's identity lives in the per-user, gitignored + // sidecar (`bd connect`'s MED-6 fix stopped mirroring it into the + // git-tracked metadata.json), so `bd context` must read it from + // there too. A missing sidecar (ErrNotConnected) or any other read + // failure just leaves ProjectID empty rather than failing this + // otherwise-degraded-state-tolerant command. + if target, err := httpclient.LoadTarget(beadsDir); err == nil { + snapshot.ProjectID = target.ExpectProjectID + } + } else { + snapshot.ProjectID = cfg.ProjectID + } if cfg.IsDoltServerMode() { snapshot.ServerHost = cfg.GetDoltServerHost() diff --git a/cmd/bd/doctor/gitignore.go b/cmd/bd/doctor/gitignore.go index 88904836e..b4c48e8e6 100644 --- a/cmd/bd/doctor/gitignore.go +++ b/cmd/bd/doctor/gitignore.go @@ -41,8 +41,12 @@ push-state.json proxied_server_client_info.json -# http backend activation sidecar (per-user server URL and identity pin) +# HTTP backend activation sidecar and per-user local metadata (per-user state, +# never tracked — the server URL and any ca_file path are machine-specific, +# and the local metadata sidecar mirrors proxied_server_client_info.json's +# per-user precedent). Written by bd connect. http_target.json +http_local_metadata.json # Worktree redirect file (contains relative path to main repo's .beads/) # Must not be committed as paths would be wrong in other clones @@ -143,6 +147,7 @@ var requiredPatterns = []string{ ".beads-credential-key", "proxied_server_client_info.json", "http_target.json", + "http_local_metadata.json", ".local_version", "backup/", } diff --git a/cmd/bd/http_backend.go b/cmd/bd/http_backend.go new file mode 100644 index 000000000..afebccdbf --- /dev/null +++ b/cmd/bd/http_backend.go @@ -0,0 +1,16 @@ +package main + +import bdhttp "github.com/steveyegge/beads/backend/http" + +// Registering the http backend is a property of the bd distribution being +// built, not of the import graph (see bdhttp.Register's own doc comment): OSS +// ships `bd serve`, so OSS `bd` should be able to talk to it too, and this is +// the one production call site. A workspace whose metadata.json names any +// OTHER unregistered backend still hard-fails with the registry's +// UnknownBackendError, exactly as before this file existed — only "http" +// becomes selectable, and only because this file registers it. +func init() { + bdhttp.Register(bdhttp.Options{ + UserAgent: "bd/" + Version + " " + bdhttp.UserAgentSuffix, + }) +} diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 901d461ee..1d3e4410c 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -30,6 +30,7 @@ import ( "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/hooks" + "github.com/steveyegge/beads/internal/httpclient" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/migration" "github.com/steveyegge/beads/internal/molecules" @@ -2757,6 +2758,28 @@ func validateWorkspaceIdentity(ctx context.Context, s storage.DoltStorage, beads return nil // No config, skip validation (fresh init) } configProjectID := cfg.ProjectID + if cfg.GetBackend() == httpclient.Backend { + // The http backend's identity pin lives in the per-user, gitignored + // sidecar, not in metadata.json: `bd connect` (MED-6) stopped + // mirroring project_id into the git-tracked config, so cfg.ProjectID + // here is whatever the workspace carried BEFORE connect — generally + // a different project's id than the server `bd connect --force` + // just pointed it at. Comparing that stale value against the live + // database's _project_id below would flag every write as drift. + // Read the sidecar instead, exactly as applyContextBackend + // (context_cmd.go) does for `bd context`. A missing sidecar + // (ErrNotConnected) or any other read failure falls through to the + // "no project_id" skip just below, the same as a pre-identity + // workspace — the wire handshake's own ExpectProjectID gate (design + // D6) is what actually refuses a mismatched server at dial time; + // this check only needs to avoid a FALSE mismatch once that gate has + // already passed. + target, err := httpclient.LoadTarget(beadsDir) + if err != nil { + return nil + } + configProjectID = target.ExpectProjectID + } if configProjectID == "" { return nil // No project_id in config (pre-identity era) } diff --git a/cmd/bd/storage_chain.go b/cmd/bd/storage_chain.go index 5ad976017..0c420d5c2 100644 --- a/cmd/bd/storage_chain.go +++ b/cmd/bd/storage_chain.go @@ -40,6 +40,17 @@ func wireStorageDecorators(store storage.DoltStorage, hookRunner *hooks.Runner, // wireExternalDependencyPolicy applies only the read/guard policy. Routed // stores must use this without inheriting the caller's hooks or telemetry. +// +// It ALWAYS wraps, including a registered Remote backend (design 3.6): a +// remote server that enforces this policy itself answers the decorator's +// storage.ExternalDependencyPolicyProber probe (implemented by +// httpclient.Store, keyed on the handshake's wire.CapExternalDependencies +// token) with true, so the decorator's own loadBlockingState is a cheap no-op +// for that server — but a server that does NOT advertise the capability must +// still get client-side enforcement, which requires wrapping it in the first +// place. Skipping the wrap here for every remote backend (the pre-S6-fix +// behavior) silently dropped the policy entirely against such a server; the +// capability-aware skip now lives inside the decorator, not at wrap time. func wireExternalDependencyPolicy(store storage.DoltStorage) storage.DoltStorage { if store == nil { return nil diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md index 9815c30b1..f00882cb5 100644 --- a/engdocs/design/http-divergence-ledger.md +++ b/engdocs/design/http-divergence-ledger.md @@ -47,7 +47,7 @@ the behavior. ## Summary -264 rows: 205 refuse, 33 degrade, 26 retired. +264 rows: 204 refuse, 34 degrade, 26 retired. ## Whole-behavior divergences (D9 ledger) @@ -62,13 +62,13 @@ The L-rows the design's D9 table names: whole behaviors that differ over HTTP, e | L5 | degrade | — | molecules are not auto-loaded pre-run | the loader writes, the write refuses, and the call site swallows the refusal to debug.Logf | D9 L5, D4 | TODO(S3): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | | L6 | degrade | — | a pre-run auto-import ATTEMPT reaches an http workspace and fails as a best-effort stderr warning; nothing is imported | THE MECHANISM THIS ROW USED TO NAME DOES NOT EXIST. It said the path was "force-disabled through an import.auto=false env override", and there is no such override anywhere: shouldRunAutoImportJSONL (cmd/bd/main.go) reads the config key, whose default is true (internal/config/config.go), and no http path sets it false. WHAT ACTUALLY HAPPENS is that maybeAutoImportJSONL runs — on any non-read-only, non-global, non-server-mode command — and is stopped by its own guards rather than by a switch: a missing or empty issues.jsonl, the attempt stamp, or the emptiness guard, which is a GetStatistics the server serves. Where all three pass (an empty server beside a non-empty local JSONL) the import is attempted and fails, because the client is not a jsonlImporter and the fallback importer's first write — SetConfig, or CreateIssuesWithFullOptions — is on the unsupported allowlist. The function is documented best-effort, so the failure is an stderr warning and the command proceeds. The DEGRADATION is therefore that nothing imports, which is the outcome the design wanted for the right reason — importing a local JSONL into a shared server is not a pre-run side effect — reached by refusal rather than by configuration. It stays a degrade because the command proceeds and the answer is not narrower or wider than the caller asked for. The retirement path is a real skip on the http backend, so the warning stops being the mechanism | D9 L6, D4 | TODO(S3): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | | L7 | degrade | — | the client-side status/type vocabulary is defaults plus the status.custom/types.custom/types.infra config keys; table-backed customs are invisible client-side | the custom_statuses/custom_types tables have no wire operation; server-side role validation still uses the true vocabulary | D9 L7, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| L8 | refuse | `--deps` | `bd list --deps` refuses, and the pretty/--format dependency-decoration arms render undecorated | GetAllDependencyRecords has no wire mapping in v1; listDependencies is an anchored read. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering | D9 L8, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L8 | degrade | `--deps` | `bd list --deps` and the policy decorator's compatibility fallback for its whole-workspace blocking state (design 3.6) both work, but by a client-side chunked scan rather than a single wire call | GetAllDependencyRecords has no 1:1 wire mapping in v1 — listDependencies is anchored at <=100 ids per call (L12's maxEdgeCountAnchors) — so the client enumerates every issue id (fetchIssuePages) and issues it in chunks, grouping the results itself; this is slower and heavier than a local store's single query, not absent. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering. The decorator takes that scan for its whole-workspace answers: ready work and its counts, the ready claim, and the single-issue close guard. Its per-issue roles (blocking annotation, the dependency tree, claim, and a batch close without a next claim) build on the client's served roles instead and read only the edges of the issues they name, through the served EdgeReader in chunks under the same cap (internal/storage/externaldeps/remote_roles.go) | D9 L8, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L9 | degrade | — | credential-bearing settings are unreadable client-side | the server redacts by omission and says so on the wire in Setting.redacted; the client answers absent-WITH-A-REASON rather than the empty string a caller would read as unset | D9 L9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L10 | refuse | `--watch` | `bd list --watch` and `bd show --watch` refuse | the loops poll every 2s, each `bd list` tick is a full cursor walk (L2), per-tick errors spam or vanish, and the decoration is refused anyway (L8); N watchers against one server is a request storm its semaphore answers with 503s the loop does not back off from | D9 L10, D10 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L11 | degrade | — | hooks fire client-side after the server's commit with no shared transaction, on connected workspaces only | the seam sits below HookFiringStore in the client process; a --server-url ephemeral invocation has no hook directory and runs none | D9 L11, D1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L12 | refuse | — | an inexpressible filter refuses, never drops — the class rule every E- row below is an instance of | a dropped filter widens a result set invisibly, and the server can only reject parameters it receives | D9 L12, D7 | TestEncoderHonorsEveryTableDisposition | | L13 | degrade | — | pretty `bd ready` shows no parent-epic context | buildParentEpicMap swallows the GetDependencyRecordsForIssues refusal into a nil map; the retirement path is an anchored listDependencies plus per-parent getIssue | D9 L13, D4 | TODO(S3): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | -| L14 | degrade | — | against a server that does NOT advertise issues.claimNext, the composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction | claimIssue validates claimability, not readiness, so a listing plus a claim is not the local role's one transaction and cannot be made into one. THE ROW IS NARROWER THAN IT WAS. Upstream published POST /v0/beads/issues:claimNext (#5510) and client wave ga-jpywb dials it, so on any server that advertises the token NONE of these three residues exists: selection, the compare-and-set and the hydration share the server's own transaction, an empty front is a 200 with `claimed` absent rather than a lost-races error, and the counts describe the state the claim produced. The whole ReadyClaimer contract tier runs against that leg with nothing parked, which is the measurement that says the residues were the COMPOSITION's rather than the wire's. WHAT KEEPS THE ROW ALIVE is the DOWN-LEVEL leg, which survives on purpose: `bd ready --claim` worked against pre-#5510 servers before this port, and refusing it now would be a regression dressed as progress. It is the posture BatchCloser takes toward issues.batchClose, and it is why this row describes shipped behavior rather than history. The third residue is stated in its IMPLEMENTED form, which is not the one the design anticipated: the design expected a follow-up getIssue to hydrate the counts, and therefore a read that could fail AFTER the claim was durable. The composition takes the counts from the ready page it already fetched instead, so that failure mode does not exist — a claim changes no dependency, dependent or comment count, and what is left is staleness bounded by the same fetch-to-dial window residue (a) already owns. Retirement is no longer an upstream ask but a fleet fact: the row goes when no server this client may meet is older than #5510 | D9 L14, D8 row 4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L14 | degrade | — | against a server that does NOT advertise issues.claimNext, or on the CLI's chain while any issue in the workspace, closed ones included, holds an unsatisfied external ref, a composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction | claimIssue validates claimability, not readiness, so a listing plus a claim is not the local role's one transaction and cannot be made into one. THE ROW IS NARROWER THAN IT WAS. Upstream published POST /v0/beads/issues:claimNext (#5510) and client wave ga-jpywb dials it, so on any server that advertises the token NONE of these three residues exists: selection, the compare-and-set and the hydration share the server's own transaction, an empty front is a 200 with `claimed` absent rather than a lost-races error, and the counts describe the state the claim produced. The whole ReadyClaimer contract tier runs against that leg with nothing parked, which is the measurement that says the residues were the COMPOSITION's rather than the wire's. WHAT KEEPS THE ROW ALIVE is the DOWN-LEVEL leg, which survives on purpose: `bd ready --claim` worked against pre-#5510 servers before this port, and refusing it now would be a regression dressed as progress. It is the posture BatchCloser takes toward issues.batchClose, and it is why this row describes shipped behavior rather than history. The third residue is stated in its IMPLEMENTED form, which is not the one the design anticipated: the design expected a follow-up getIssue to hydrate the counts, and therefore a read that could fail AFTER the claim was durable. The composition takes the counts from the ready page it already fetched instead, so that failure mode does not exist — a claim changes no dependency, dependent or comment count, and what is left is staleness bounded by the same fetch-to-dial window residue (a) already owns. A SECOND COMPOSED LEG exists on any server. On the CLI's chain the external-dependency decorator (design 3.6) hands the claim to this role only while no issue in the workspace holds an unsatisfied external ref, because claimNext has no parameter to exclude what that policy refuses. Otherwise it claims down its own filtered ready page through the served Claimer, so all three residues return, the second as a lost-races error (internal/storage/externaldeps/remote_roles.go). A closed holder counts, as it does in every backend's blocking state: it can never be a candidate, so the composed claim picks what the served one would, and leaving it out would cost a status read per holder. A server that advertises wire.CapExternalDependencies enforces the policy itself and never reaches that leg. `bd close --claim-next` takes no composed leg: the decorator refuses it rather than claim after the close commits (F-close). Retirement of the down-level leg is no longer an upstream ask but a fleet fact: it goes when no server this client may meet is older than #5510. The decorator's leg goes with an exclusion parameter on claimNext, or once every server enforces the policy itself | D9 L14, D8 row 4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L15 | degrade | — | a --max-rows cap can fire over http on a query whose Limit is at or under the cap, on the `bd list` legs that still walk | locally the effective SQL LIMIT is min(Limit, MaxRows+1), so a window at or under the cap can never exceed it; over http a walk to exhaustion fetches far more than the window and the cap deliberately bounds WIRE ROWS FETCHED — safety wins over exit-code parity on exactly the unbounded walk the cap exists to bound. NARROWED by the same pushdown that narrowed L2: the pushdown leg asks for min(limit, MaxRows+1) rows in one request, which IS the local expression, so a cap at or above the limit can no longer fire there and a request that used to exit 2 now answers. That is a convergence toward the contract, and it is observable — the same command changes from a refusal to a page. THE NARROWING IS SCOPED TO THE ORDERS SQL CAN EXPRESS, and that scope is a correctness bound rather than an optimization left on the table. For a GO-SIDE sort — sqlbuild.IsGoSideSort, today `--sort id`, which needs the natural-numeric comparison (bd-9 before bd-10) no ORDER BY renders — workapi.SQLLimit pushes 0 down instead of the limit, so LOCALLY the window is MaxRows+1 whatever the limit is and the cap fires on the overage even under a limit at or below it. min(limit, MaxRows+1) is therefore NOT the local expression there, and a request bounded by the limit cannot see the overage at all. walkIssues' gate keeps a capped Go-side sort on the walk, where fetching MaxRows+1 wire rows reproduces that unbounded local window exactly and raises the same refusal — so on this one shape the walk leg is the CONVERGENT one and pushing down would have been the divergence. What keeps the row live is the walk legs (unlimited reads, keyset positions, down-level servers), where the old accounting is unchanged and correct, and `bd ready`, which has no pushdown at all | D9 L15, D12 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L16 | refuse | `--file` | `bd dep add --file` carrying more than 100 edges refuses, naming the wire bound; it is never chunked | maxAddDependencyEdges = 100 (internal/httpapi/dependency_edit.go); chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract, where naive forwarding at least fails loudly | D9 L16, D7, D8 row 18 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L-close-cap | refuse | — | a batch close naming more than 100 items refuses, naming the wire bound; it is never chunked | maxBatchCloseItems = 100 (internal/httpapi/batch_close.go); the item cap bounds how long one request may hold a write transaction. Chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract — the same reason L16 refuses a bulk dependency add rather than splitting it — so a larger list refuses and the caller splits it, each request atomic on its own | D9 L-close-cap, D7 F-close, D8 row 17 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | @@ -157,7 +157,7 @@ Whole commands, and the flag modes of served commands, that dispatch onto method | F-blocked | refuse | — | `bd blocked` refuses | it is raw GetBlockedIssues, which is on the unsupported allowlist | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | F-count | retired | — | RETIRED — `bd count` is SERVED whole | it said the command refuses because "issueops.Counter has no wire operation; a counts operation is a filed upstream ask", and that ask was DELIVERED: upstream #5508 published countIssues, client wave ga-icks1 wired the Counter accessor, and `count` moved from httpRefusedCommands into httpServedCommands. D8 row 20 and docs/reference/http-workspaces.md have said so since; this row was the last place that did not. Every flag the command publishes reaches a wire member — the whole predicate, the six instant bounds, the three emptiness checks, --include-infra, and the five --by-* flags as the one group_by parameter — so there is no refused mode left to name either. HOW IT ROTTED IS THE PART WORTH KEEPING. Its pin was TestClassifiedRefusedCommandsRefuseBeforeTheOSSPreRun, which walks httpRefusedCommands and drives every entry. The moment `count` left that table the pin stopped covering this row — it stayed LIVE BY NAME, so the well-formedness gate that resolves PinnedBy to a real test kept passing, and DEAD BY COVERAGE, so nothing anywhere asserted the row's claim. A ledger row can be false for a whole wave under a green pin, which is what TestFRowsAgreeWithTheCommandClassification now makes impossible | D7 (RETIRED), D8 row 20 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | F-serve | refuse | — | `bd serve` refuses against an http workspace | the OSS classifier would happily serve it from the opened store, and a bd serve re-serving a remote bd serve is a proxy chain nobody designed: run serve where the database is | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| F-close | refuse | — | `bd close` is SERVED whole — single id, several ids and --claim-next — over issues:batchClose; it refuses only against a DOWN-LEVEL server that does not advertise issues.batchClose | the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so the whole CloseBatchRequest — multi-item and the atomic ClaimNext included — is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), and cmd/bd's renderEscapedHTTPRefusal turns it into the capability-absent taxonomy (case 2) by scanning the served command's own capability list for the first token the server does not advertise — issues.batchClose, which is the token it then names to upgrade to. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap | D8 row 17 (resolved, Wire), D9 L18, D9 L-close-cap | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-close | refuse | — | `bd close` is SERVED whole — single id and several ids — over issues:batchClose; it refuses against a DOWN-LEVEL server that does not advertise issues.batchClose, and --claim-next refuses against every server, before anything closes | the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so a CloseBatchRequest of any size is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. THE NEXT CLAIM IS THE EXCEPTION. batchCloseIssues has no member to carry it, so the client refuses a ClaimNext on both legs, after the claim's own validation and before the dial (W-CloseBatchRequest.ClaimNext). Closing and then claiming would split the role's one transaction in two, and a claim that failed after the closes committed could not report them, because the role answers with an error or with outcomes, never both. On the CLI's chain the external-dependency decorator (design 3.6) refuses first unless the server advertises wire.CapExternalDependencies, and its refusal says nothing closed and names the commands that do the same work: close without --claim-next, then `bd ready --claim` (internal/storage/externaldeps/remote_roles.go). Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), which names the refused shape and the server, and cmd/bd prints it against each id the batch carried. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap | D8 row 17 (resolved, Wire), D9 L18, D9 L-close-cap | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | F-partial-id | refuse | — | partial-id resolution refuses with its own taxonomy text | SearchIssueIDs has no wire operation, and the client cannot tell a partial id from a full id that does not exist — so the refusal text covers both outcomes rather than falling through to a raw search error | D11 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | ## Ready-request fields diff --git a/internal/gitignore/BUILD.bazel b/internal/gitignore/BUILD.bazel index 1f3a63467..8f2a4c81d 100644 --- a/internal/gitignore/BUILD.bazel +++ b/internal/gitignore/BUILD.bazel @@ -2,14 +2,20 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "gitignore", - srcs = ["append.go"], + srcs = [ + "append.go", + "ensure.go", + ], importpath = "github.com/steveyegge/beads/internal/gitignore", visibility = ["//:__subpackages__"], ) go_test( name = "gitignore_test", - srcs = ["append_test.go"], + srcs = [ + "append_test.go", + "ensure_test.go", + ], embed = [":gitignore"], ) diff --git a/internal/gitignore/ensure.go b/internal/gitignore/ensure.go new file mode 100644 index 000000000..7d5795cc6 --- /dev/null +++ b/internal/gitignore/ensure.go @@ -0,0 +1,42 @@ +package gitignore + +import ( + "os" + "path/filepath" + "strings" +) + +// EnsurePatternIgnored appends pattern to dir/.gitignore as a standalone line +// if it is not already present (matched exactly, after trimming whitespace, +// against each existing line — the same check +// cmd/bd/doctor.EnsureGitignoreForBeadsDir uses for its own required +// patterns). It creates the file (0600) when it does not exist yet. +// +// This is the minimal sibling of doctor.EnsureGitignoreForBeadsDir, not a +// move of it: that function also owns the full canonical .beads/.gitignore +// template, repairs permissions on an existing file, and covers every +// pattern bd's own on-disk state needs — and the cmd/bd/doctor package as a +// whole pulls in dolt and git-process dependencies through its other files. +// A caller like backend/http, which promises an embedder a minimal dependency +// footprint, cannot import cmd/bd/doctor just for this one line without +// breaking that promise. EnsurePatternIgnored exists so such a caller can +// still make the one guarantee it actually needs — this pattern is never +// offered to `git add` — on its own. The two are complementary and +// idempotent together: a workspace that goes through both ends up with the +// same file either way. +func EnsurePatternIgnored(dir, pattern string) error { + path := filepath.Join(dir, ".gitignore") + content, err := os.ReadFile(path) // #nosec G304 -- caller supplies the active beads dir + if os.IsNotExist(err) { + return os.WriteFile(path, AppendLines(nil, []string{pattern}), 0o600) + } + if err != nil { + return err + } + for _, line := range strings.Split(string(content), "\n") { + if strings.TrimSpace(line) == pattern { + return nil + } + } + return os.WriteFile(path, AppendLines(content, []string{pattern}), 0o600) +} diff --git a/internal/gitignore/ensure_test.go b/internal/gitignore/ensure_test.go new file mode 100644 index 000000000..f1b73e7d7 --- /dev/null +++ b/internal/gitignore/ensure_test.go @@ -0,0 +1,64 @@ +package gitignore + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestEnsurePatternIgnored(t *testing.T) { + t.Run("creates the file when missing", func(t *testing.T) { + dir := t.TempDir() + if err := EnsurePatternIgnored(dir, "http_target.json"); err != nil { + t.Fatalf("EnsurePatternIgnored: %v", err) + } + body, err := os.ReadFile(filepath.Join(dir, ".gitignore")) + if err != nil { + t.Fatalf("read .gitignore: %v", err) + } + if string(body) != "http_target.json\n" { + t.Errorf(".gitignore = %q, want %q", body, "http_target.json\n") + } + }) + + t.Run("appends when the pattern is missing from an existing file", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, ".gitignore") + if err := os.WriteFile(path, []byte("local/\n"), 0o600); err != nil { + t.Fatalf("seed .gitignore: %v", err) + } + if err := EnsurePatternIgnored(dir, "http_target.json"); err != nil { + t.Fatalf("EnsurePatternIgnored: %v", err) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read .gitignore: %v", err) + } + if !strings.Contains(string(body), "local/") { + t.Errorf(".gitignore lost its existing content: %q", body) + } + if !strings.Contains(string(body), "http_target.json") { + t.Errorf(".gitignore = %q, want it to contain http_target.json", body) + } + }) + + t.Run("is a no-op when the pattern is already present", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, ".gitignore") + original := "local/\nhttp_target.json\n" + if err := os.WriteFile(path, []byte(original), 0o600); err != nil { + t.Fatalf("seed .gitignore: %v", err) + } + if err := EnsurePatternIgnored(dir, "http_target.json"); err != nil { + t.Fatalf("EnsurePatternIgnored: %v", err) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read .gitignore: %v", err) + } + if string(body) != original { + t.Errorf(".gitignore changed on a no-op call: got %q, want unchanged %q", body, original) + } + }) +} diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 0a733fb39..5725230b0 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "dial.go", "dispatch.go", "errors.go", + "insecure_credential_guard.go", "journal.go", "leasereclaimer.go", "lifecycle.go", @@ -32,6 +33,7 @@ go_library( "metadata.go", "metadatacas.go", "offrole.go", + "open_with.go", "order.go", "readyclaimer.go", "refusal.go", @@ -108,10 +110,12 @@ go_test( "divergence_test.go", "expected_revision_contract_test.go", "helpers_test.go", + "insecure_credential_guard_test.go", "lifecycle_create_test.go", "lifecycle_update_test.go", "list_sort_test.go", "metadatacas_test.go", + "open_with_test.go", "race_detector_off_test.go", "race_detector_on_test.go", "required_served_tier_test.go", @@ -131,6 +135,7 @@ go_test( "served_detail_reads_test.go", "served_dualrun_test.go", "served_dualwrite_history_test.go", + "served_external_dependency_policy_test.go", "served_graph_aux_test.go", "served_graph_counter_test.go", "served_harness_test.go", @@ -190,6 +195,7 @@ go_test( "//internal/storage/batchfixtures", "//internal/storage/domain", "//internal/storage/embeddeddolt", + "//internal/storage/externaldeps", "//internal/storage/issueops", "//internal/storage/sqlbuild", "//internal/types", diff --git a/internal/httpclient/completeness_test.go b/internal/httpclient/completeness_test.go index 36ee05cd3..4e0d404d4 100644 --- a/internal/httpclient/completeness_test.go +++ b/internal/httpclient/completeness_test.go @@ -250,7 +250,6 @@ var legitimatelyUnsupported = map[string]string{ // refuse on these, which is why they are named in D7's refused class. "DetectCycles": "raw read: role CycleDetector serves this shape", "FindWispDependentsRecursive": "raw read: no recursive-dependents operation", - "GetAllDependencyRecords": "raw read: listDependencies is anchored (L8, `bd list --deps`)", "GetBlockedIssues": "raw read: no blocked-set operation (`bd blocked`)", "GetBlockingInfoForIssues": "raw read: role BlockingAnnotator serves this shape", "GetDependencies": "raw read: role EdgeReader serves this shape", diff --git a/internal/httpclient/credential.go b/internal/httpclient/credential.go index 7be27ad8f..39959e428 100644 --- a/internal/httpclient/credential.go +++ b/internal/httpclient/credential.go @@ -206,16 +206,25 @@ func (p *BearerProvider) resolve(ctx context.Context) (token, source string, err return cred.Value, cred.Source, nil } -// warnInsecure fires once per provider, and only when a token is actually about -// to travel. bd serve has no TLS of its own, so a bearer bound anywhere but -// loopback crosses the network in the clear; that is an operator decision to -// make knowingly, in front of a reverse proxy that terminates TLS. +// warnInsecure fires once per provider, and only when a token is actually +// configured and about to be offered to Authorize's caller. bd serve has no +// TLS of its own, so a bearer bound anywhere but loopback crosses the network +// in the clear if it is sent at all; that is an operator decision to make +// knowingly, in front of a reverse proxy that terminates TLS. +// +// This fires BEFORE guardInsecureCredential (insecure_credential_guard.go) +// gets a chance to refuse the request outright on an unallowed target +// (bee-ghosttrack CHANGES_REQUESTED on #7288, should-fix 2's warning-text +// half): the two layers do not coordinate, so the wording here must stay +// true on EITHER outcome rather than asserting the token is being sent — a +// claim that used to read as misleading immediately above a refusal error +// saying the opposite. func (p *BearerProvider) warnInsecure() { if !p.insecure { return } p.warnOnce.Do(func() { - fmt.Fprintf(p.warnTo, "Warning: sending a bearer token to %s over plain http; the token crosses the network in the clear. Put bd serve behind TLS, or bind it to loopback.\n", p.endpoint) + fmt.Fprintf(p.warnTo, "Warning: a bearer token is configured for %s, which is plain http and not loopback; the token would cross the network in the clear unless refused. Put bd serve behind TLS, bind it to loopback, or accept the risk knowingly (bd connect --allow-plaintext, or BEADS_HTTP_ALLOW_INSECURE=1).\n", p.endpoint) }) } diff --git a/internal/httpclient/dial.go b/internal/httpclient/dial.go index f314dcbc7..e49979d7f 100644 --- a/internal/httpclient/dial.go +++ b/internal/httpclient/dial.go @@ -72,6 +72,17 @@ type DialOptions struct { // an external HTTPClient always goes through one of the two constructors // above, or the normal env-aware resolution. caOverride *resolvedCA + + // AllowInsecureCredential opts THIS dial into sending a credential over + // plain http to a non-loopback target (MED-4, S6 review). Left false, + // DialWith refuses to let a credential cross such a target at all — see + // guardInsecureCredential. connect.go sets this from --allow-plaintext so + // its own Handshake probe honors the same opt-in the CLI flag already + // granted; every other caller defaults to false and relies on the + // process-wide BEADS_HTTP_ALLOW_INSECURE=1 escape hatch instead, since + // Open/OpenReadOnly/OpenWith's callers have no flag of their own to set + // this through. + AllowInsecureCredential bool } // Conn is the store's view of one wire client. @@ -193,6 +204,17 @@ func DialWith(target Target, creds CredentialProvider, opts DialOptions) (*Conn, // refuse. } + // target.AllowInsecureCredential (bee-ghosttrack CHANGES_REQUESTED on + // #7288, should-fix 2) is `bd connect --allow-plaintext`'s grant, + // persisted to the sidecar and scoped to the target it names: it is + // loaded back from there on every ordinary command's dial, not only + // connect's own Handshake probe, so a workspace that connected with the + // flag does not also need BEADS_HTTP_ALLOW_INSECURE=1 set for every + // later `bd` invocation. opts.AllowInsecureCredential stays OR'd in + // alongside it: connect.go sets that field for its own probe from the + // SAME flag before the sidecar carrying it even exists yet. + creds = guardInsecureCredential(target, creds, opts.AllowInsecureCredential || target.AllowInsecureCredential) + client, err := wire.New(target.BaseURL, creds, wire.Options{ HTTPClient: opts.HTTPClient, UserAgent: opts.UserAgent, diff --git a/internal/httpclient/dial_test.go b/internal/httpclient/dial_test.go index 2de4d70d7..fe537aa40 100644 --- a/internal/httpclient/dial_test.go +++ b/internal/httpclient/dial_test.go @@ -181,6 +181,11 @@ func TestDialOptionsForFileBypassesEnvEntirely(t *testing.T) { if err != nil { t.Fatalf("DialOptionsForFile: %v", err) } + // u names the server as "localhost" (so SNI is sent) even though + // startServer actually bound 127.0.0.1; without pinning the dial to + // tcp4, the default dual-stack dialer can race an unrelated ::1 listener + // and flake. See forceIPv4Loopback's doc for the full mechanism. + forceIPv4Loopback(t, opts.HTTPClient.Transport) // Target.CAFile is deliberately left empty, mirroring cmd/bd's connect // verification call. target := Target{BaseURL: u} diff --git a/internal/httpclient/encode/ledger.go b/internal/httpclient/encode/ledger.go index f40002da3..6952cdb9d 100644 --- a/internal/httpclient/encode/ledger.go +++ b/internal/httpclient/encode/ledger.go @@ -320,9 +320,10 @@ func designRows() []Row { PinnedBy: pinnedByS3Conformance, }, { - ID: "L8", Kind: KindRefuse, Flag: "--deps", - What: "`bd list --deps` refuses, and the pretty/--format dependency-decoration arms render undecorated", - Why: "GetAllDependencyRecords has no wire mapping in v1; listDependencies is an anchored read. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering", + ID: "L8", Kind: KindDegrade, Flag: "--deps", + What: "`bd list --deps` and the policy decorator's compatibility fallback for its whole-workspace blocking state (design 3.6) both work, but by a client-side chunked scan rather than a single wire call", + Why: "GetAllDependencyRecords has no 1:1 wire mapping in v1 — listDependencies is anchored at <=100 ids per call (L12's maxEdgeCountAnchors) — so the client enumerates every issue id (fetchIssuePages) and issues it in chunks, grouping the results itself; this is slower and heavier than a local store's single query, not absent. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering. " + + "The decorator takes that scan for its whole-workspace answers: ready work and its counts, the ready claim, and the single-issue close guard. Its per-issue roles (blocking annotation, the dependency tree, claim, and a batch close without a next claim) build on the client's served roles instead and read only the edges of the issues they name, through the served EdgeReader in chunks under the same cap (internal/storage/externaldeps/remote_roles.go)", SpecRow: "D9 L8, D4", PinnedBy: pinnedByS3Conformance, }, @@ -368,12 +369,13 @@ func designRows() []Row { }, { ID: "L14", Kind: KindDegrade, - What: "against a server that does NOT advertise issues.claimNext, the composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction", + What: "against a server that does NOT advertise issues.claimNext, or on the CLI's chain while any issue in the workspace, closed ones included, holds an unsatisfied external ref, a composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction", Why: "claimIssue validates claimability, not readiness, so a listing plus a claim is not the local role's one transaction and cannot be made into one. " + "THE ROW IS NARROWER THAN IT WAS. Upstream published POST /v0/beads/issues:claimNext (#5510) and client wave ga-jpywb dials it, so on any server that advertises the token NONE of these three residues exists: selection, the compare-and-set and the hydration share the server's own transaction, an empty front is a 200 with `claimed` absent rather than a lost-races error, and the counts describe the state the claim produced. The whole ReadyClaimer contract tier runs against that leg with nothing parked, which is the measurement that says the residues were the COMPOSITION's rather than the wire's. " + "WHAT KEEPS THE ROW ALIVE is the DOWN-LEVEL leg, which survives on purpose: `bd ready --claim` worked against pre-#5510 servers before this port, and refusing it now would be a regression dressed as progress. It is the posture BatchCloser takes toward issues.batchClose, and it is why this row describes shipped behavior rather than history. " + "The third residue is stated in its IMPLEMENTED form, which is not the one the design anticipated: the design expected a follow-up getIssue to hydrate the counts, and therefore a read that could fail AFTER the claim was durable. The composition takes the counts from the ready page it already fetched instead, so that failure mode does not exist — a claim changes no dependency, dependent or comment count, and what is left is staleness bounded by the same fetch-to-dial window residue (a) already owns. " + - "Retirement is no longer an upstream ask but a fleet fact: the row goes when no server this client may meet is older than #5510", + "A SECOND COMPOSED LEG exists on any server. On the CLI's chain the external-dependency decorator (design 3.6) hands the claim to this role only while no issue in the workspace holds an unsatisfied external ref, because claimNext has no parameter to exclude what that policy refuses. Otherwise it claims down its own filtered ready page through the served Claimer, so all three residues return, the second as a lost-races error (internal/storage/externaldeps/remote_roles.go). A closed holder counts, as it does in every backend's blocking state: it can never be a candidate, so the composed claim picks what the served one would, and leaving it out would cost a status read per holder. A server that advertises wire.CapExternalDependencies enforces the policy itself and never reaches that leg. `bd close --claim-next` takes no composed leg: the decorator refuses it rather than claim after the close commits (F-close). " + + "Retirement of the down-level leg is no longer an upstream ask but a fleet fact: it goes when no server this client may meet is older than #5510. The decorator's leg goes with an exclusion parameter on claimNext, or once every server enforces the policy itself", SpecRow: "D9 L14, D8 row 4", PinnedBy: pinnedByS3Conformance, }, @@ -1102,9 +1104,10 @@ func commandRows() []Row { "the OSS classifier would happily serve it from the opened store, and a bd serve re-serving a remote bd serve is a proxy chain nobody designed: run serve where the database is"), { ID: "F-close", Kind: KindRefuse, Command: "close", Capability: "issues.batchClose", - What: "`bd close` is SERVED whole — single id, several ids and --claim-next — over issues:batchClose; it refuses only against a DOWN-LEVEL server that does not advertise issues.batchClose", - Why: "the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so the whole CloseBatchRequest — multi-item and the atomic ClaimNext included — is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. " + - "Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), and cmd/bd's renderEscapedHTTPRefusal turns it into the capability-absent taxonomy (case 2) by scanning the served command's own capability list for the first token the server does not advertise — issues.batchClose, which is the token it then names to upgrade to. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap", + What: "`bd close` is SERVED whole — single id and several ids — over issues:batchClose; it refuses against a DOWN-LEVEL server that does not advertise issues.batchClose, and --claim-next refuses against every server, before anything closes", + Why: "the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so a CloseBatchRequest of any size is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. " + + "THE NEXT CLAIM IS THE EXCEPTION. batchCloseIssues has no member to carry it, so the client refuses a ClaimNext on both legs, after the claim's own validation and before the dial (W-CloseBatchRequest.ClaimNext). Closing and then claiming would split the role's one transaction in two, and a claim that failed after the closes committed could not report them, because the role answers with an error or with outcomes, never both. On the CLI's chain the external-dependency decorator (design 3.6) refuses first unless the server advertises wire.CapExternalDependencies, and its refusal says nothing closed and names the commands that do the same work: close without --claim-next, then `bd ready --claim` (internal/storage/externaldeps/remote_roles.go). " + + "Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), which names the refused shape and the server, and cmd/bd prints it against each id the batch carried. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap", SpecRow: "D8 row 17 (resolved, Wire), D9 L18, D9 L-close-cap", // The command is served; the down-level refusal is the capability gate's, // exercised against a server without issues.batchClose. diff --git a/internal/httpclient/insecure_credential_guard.go b/internal/httpclient/insecure_credential_guard.go new file mode 100644 index 000000000..07dc0e3e9 --- /dev/null +++ b/internal/httpclient/insecure_credential_guard.go @@ -0,0 +1,116 @@ +// Written fresh for OSS beads S6 (no bd-enterprise source copied): closes +// the S6 review's MED-4 finding. BearerProvider.warnInsecure (credential.go) +// only ever WARNED that a bearer was bound for a non-loopback http:// target; +// `bd connect` (connect.go) separately REFUSES that combination, but only at +// connect time, and only for the ambient ladder it dials with there. Every +// ordinary open after that — Open, OpenReadOnly (the registered dialer) and +// OpenWith (the per-open seam backend/http's embedder door uses) — had no +// refusal at all: a sidecar or an embedder-supplied Target pinned at a +// non-loopback http:// host, however that happened, would have every later +// `bd` command send its bearer in the clear with nothing stopping it. +package httpclient + +import ( + "context" + "fmt" + "net/http" + "os" + "strings" + + "github.com/steveyegge/beads/internal/configfile" +) + +// AllowInsecureCredentialEnv opts an entire process into sending a credential +// over plain http to a non-loopback bd serve. +// +// It exists because Open/OpenReadOnly/OpenWith's callers — every bd command +// but connect — have no flag of their own to thread an opt-in through, unlike +// `bd connect --allow-plaintext`. Setting it is a deliberate, informed +// decision (the same one --allow-plaintext records at connect time): the +// operator trusts the network path, typically a reverse proxy terminating TLS +// in front of bd serve. +// +// #nosec G101 -- the NAME of an environment variable, not a credential. +const AllowInsecureCredentialEnv = "BEADS_HTTP_ALLOW_INSECURE" + +func allowInsecureCredentialFromEnv() bool { + v := strings.TrimSpace(os.Getenv(AllowInsecureCredentialEnv)) + return v == "1" || strings.EqualFold(v, "true") +} + +// guardInsecureCredential wraps creds so that, once bound for a plain-http, +// non-loopback target, it refuses to let ANY credential header it adds reach +// the wire — unless allowed, which is true when EITHER opts.AllowInsecureCredential +// (DialOptions, which connect.go's own Handshake probe sets from +// --allow-plaintext before any sidecar exists) OR target.AllowInsecureCredential +// (the sidecar's persisted record of that same grant, read back on every +// LATER dial for this workspace — see Target.AllowInsecureCredential's own +// doc) is true, or BEADS_HTTP_ALLOW_INSECURE=1 is set in the environment. +// +// It wraps the resolved wire.CredentialProvider itself, never switching on +// its concrete type, so the refusal is uniform across the ambient +// BearerProvider ladder (Dial, Open, OpenReadOnly) AND an embedder-supplied +// ProvidedCredential (backends.OpenOptions.Credential, OpenWith's seam) — any +// implementation at all, including third-party ones this package has never +// heard of. +// +// A nil creds (the tip server's loopback-trust posture: no Authorization +// header is ever sent for an unauthenticated open) passes through untouched — +// there is nothing to guard, and refusing an unauthenticated plaintext dial +// would be a different, broader policy than the one this finding asks for +// ("refuse SENDING CREDENTIALS over http:// to non-loopback"). +func guardInsecureCredential(target Target, creds CredentialProvider, allowed bool) CredentialProvider { + if creds == nil { + return nil + } + base := target.BaseURL + if base == nil || base.Scheme != "http" || configfile.IsLocalHostString(base.Hostname()) { + return creds + } + if allowed || allowInsecureCredentialFromEnv() { + return creds + } + return &insecureCredentialGuard{inner: creds, endpoint: base.Redacted()} +} + +// insecureCredentialGuard is guardInsecureCredential's refusal. It lets the +// inner provider run first (so a provider that attaches nothing — an +// unconfigured ambient ladder — is never refused) and only turns the request +// into an error when Authorize actually added a header: the diff is taken on +// the header count rather than on any named header, deliberately, since +// CredentialProvider's own doc names more than one scheme ("Authorization: +// Bearer, DPoP, ...") and a third-party provider may add a header this +// package has never heard of either. +type insecureCredentialGuard struct { + inner CredentialProvider + endpoint string +} + +func (g *insecureCredentialGuard) Authorize(ctx context.Context, req *http.Request) error { + before := len(req.Header) + if err := g.inner.Authorize(ctx, req); err != nil { + return err + } + if len(req.Header) > before { + return fmt.Errorf( + "refusing to send a credential to %s over plain http: %s is not loopback, and the credential would cross the network unencrypted; use https://, run `bd connect --allow-plaintext` to accept the risk knowingly for this server, or set %s=1 to override", + g.endpoint, req.URL.Hostname(), AllowInsecureCredentialEnv) + } + return nil +} + +// Refresh delegates unchanged: a refused request never reaches a 401, so this +// guard has nothing of its own to add here. +func (g *insecureCredentialGuard) Refresh(ctx context.Context) (bool, error) { + return g.inner.Refresh(ctx) +} + +// Source forwards the inner provider's provenance report, structurally +// satisfying wire's unexported credentialSourceReporter without importing it, +// so a 401 after an ALLOWED insecure dial still names the right ladder rung. +func (g *insecureCredentialGuard) Source() string { + if r, ok := g.inner.(interface{ Source() string }); ok { + return r.Source() + } + return "" +} diff --git a/internal/httpclient/insecure_credential_guard_test.go b/internal/httpclient/insecure_credential_guard_test.go new file mode 100644 index 000000000..8e821f403 --- /dev/null +++ b/internal/httpclient/insecure_credential_guard_test.go @@ -0,0 +1,251 @@ +// Written fresh for OSS beads S6 (no bd-enterprise source copied). Tests +// guardInsecureCredential (insecure_credential_guard.go), the S6 review's +// MED-4 fix: Open/OpenReadOnly/OpenWith's shared choke point for refusing a +// credential bound for a plain-http, non-loopback target. +package httpclient + +import ( + "context" + "net/http" + "strings" + "testing" +) + +const nonLoopbackTestHost = "198.51.100.1" // RFC 5737 TEST-NET-2: reserved, globally unroutable. + +func newTestRequest(t *testing.T, rawURL string) *http.Request { + t.Helper() + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + return req +} + +// TestGuardInsecureCredentialPassesThroughNilCredential pins the loopback- +// trust posture: an unauthenticated open never had anything to guard, and +// must not be turned into a refusal by this fix. +func TestGuardInsecureCredentialPassesThroughNilCredential(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + if got := guardInsecureCredential(target, nil, false); got != nil { + t.Fatalf("guardInsecureCredential(nil) = %v, want nil", got) + } +} + +// TestGuardInsecureCredentialPassesThroughForLoopback: the finding is about +// a NON-loopback target; a loopback one (what the tip OSS server ships with) +// must be unaffected regardless of opts. +func TestGuardInsecureCredentialPassesThroughForLoopback(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://127.0.0.1:8080/")} + inner := stubProvider{tag: "loopback"} + got := guardInsecureCredential(target, inner, false) + if got != CredentialProvider(inner) { + t.Fatalf("guardInsecureCredential returned %T, want the inner provider unwrapped for a loopback target", got) + } +} + +// TestGuardInsecureCredentialPassesThroughForHTTPS: the finding is about +// PLAIN http; a non-loopback https target is exactly what a credential should +// cross the network over, so it must be unaffected. +func TestGuardInsecureCredentialPassesThroughForHTTPS(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "https://"+nonLoopbackTestHost+"/")} + inner := stubProvider{tag: "https"} + got := guardInsecureCredential(target, inner, false) + if got != CredentialProvider(inner) { + t.Fatalf("guardInsecureCredential returned %T, want the inner provider unwrapped for an https target", got) + } +} + +// TestGuardInsecureCredentialRefusesNonLoopbackPlainHTTP is MED-4's core +// case: a credential provider that DOES attach a header to a non-loopback +// plain-http target must be refused, by default, with no opt-in given. +func TestGuardInsecureCredentialRefusesNonLoopbackPlainHTTP(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + inner := stubProvider{tag: "will-attach"} + guarded := guardInsecureCredential(target, inner, false) + if guarded == CredentialProvider(inner) { + t.Fatal("guardInsecureCredential returned the inner provider unwrapped for a non-loopback plain-http target") + } + + req := newTestRequest(t, "http://"+nonLoopbackTestHost+"/v0/beads/context") + err := guarded.Authorize(context.Background(), req) + if err == nil { + t.Fatal("Authorize succeeded; want a refusal (plain http, non-loopback, no opt-in)") + } + if !strings.Contains(err.Error(), "refusing to send a credential") || !strings.Contains(err.Error(), AllowInsecureCredentialEnv) { + t.Errorf("Authorize error = %q, want it to name the refusal and %s", err.Error(), AllowInsecureCredentialEnv) + } +} + +// TestGuardInsecureCredentialPassesThroughWhenNothingIsAttached: a provider +// that is consulted but attaches NOTHING (the ambient ladder with no token +// configured, the tip server's loopback-trust posture extended to a +// non-loopback target with no credential at all) must not be refused — the +// finding is specifically about a credential crossing the network, not about +// plain http to a non-loopback host in general. +func TestGuardInsecureCredentialPassesThroughWhenNothingIsAttached(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + inner := noopProvider{} + guarded := guardInsecureCredential(target, inner, false) + + req := newTestRequest(t, "http://"+nonLoopbackTestHost+"/v0/beads/context") + if err := guarded.Authorize(context.Background(), req); err != nil { + t.Fatalf("Authorize: %v, want no refusal since no credential was attached", err) + } + if req.Header.Get("Authorization") != "" { + t.Fatalf("Authorization header = %q, want none", req.Header.Get("Authorization")) + } +} + +// TestGuardInsecureCredentialAllowedByOptIn proves DialOptions' +// AllowInsecureCredential=true (what connect.go sets from --allow-plaintext) +// lets the SAME combination through that the default test above refuses. +func TestGuardInsecureCredentialAllowedByOptIn(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + inner := stubProvider{tag: "opted-in"} + guarded := guardInsecureCredential(target, inner, true) + if guarded != CredentialProvider(inner) { + t.Fatalf("guardInsecureCredential(allowed=true) returned %T, want the inner provider unwrapped", guarded) + } +} + +// TestGuardInsecureCredentialAllowedByEnv proves the BEADS_HTTP_ALLOW_INSECURE +// escape hatch Open/OpenReadOnly/OpenWith's callers rely on (they have no +// flag of their own to set opts.AllowInsecureCredential through). +func TestGuardInsecureCredentialAllowedByEnv(t *testing.T) { + t.Setenv(AllowInsecureCredentialEnv, "1") + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + inner := stubProvider{tag: "env-opted-in"} + guarded := guardInsecureCredential(target, inner, false) + if guarded != CredentialProvider(inner) { + t.Fatalf("guardInsecureCredential(env opt-in) returned %T, want the inner provider unwrapped", guarded) + } +} + +// TestGuardInsecureCredentialDelegatesRefreshAndSource proves the refusal +// wrapper stays transparent to the two capabilities a CredentialProvider may +// also offer, so a 401's rotation retry and its source-naming both still work +// across an ALLOWED insecure dial. +func TestGuardInsecureCredentialDelegatesRefreshAndSource(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + inner := &reportingProvider{source: "test-rung"} + guarded := guardInsecureCredential(target, inner, true) + if guarded != CredentialProvider(inner) { + t.Fatalf("guardInsecureCredential(allowed=true) returned %T, want the inner provider unwrapped", guarded) + } + + // With allowed=false the guard wraps, and must still forward both calls. + wrapped := guardInsecureCredential(target, inner, false) + retry, err := wrapped.Refresh(context.Background()) + if err != nil || !retry { + t.Fatalf("Refresh = (%v, %v), want (true, nil)", retry, err) + } + reporter, ok := wrapped.(interface{ Source() string }) + if !ok { + t.Fatal("wrapped provider does not implement Source() string") + } + if got := reporter.Source(); got != "test-rung" { + t.Fatalf("Source() = %q, want %q", got, "test-rung") + } +} + +// TestDialWithWiresAllowInsecureCredentialThroughToTheGuard is the +// integration proof that DialWith actually applies guardInsecureCredential +// (not just that the helper is correct in isolation): a real ambient +// BearerProvider (BEADS_HTTP_TOKEN set) dialing a non-loopback plain-http +// target must be refused before the server ever sees the request, and the +// SAME dial must go through once AllowInsecureCredential is set — proving +// connect.go's --allow-plaintext -> DialOptions.AllowInsecureCredential wire +// actually reaches this guard. +func TestDialWithWiresAllowInsecureCredentialThroughToTheGuard(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, nonLoopbackTestHost+"=integration-token") + server := &contextServer{body: v0Context("proj-insecure-wiring")} + srv := server.start(t) + + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + redirecting := &http.Client{Transport: &redirectToAddrTransport{addr: strings.TrimPrefix(srv.URL, "http://")}} + + _, err := Handshake(context.Background(), target, DialOptions{HTTPClient: redirecting}) + if err == nil { + t.Fatal("Handshake succeeded; want DialWith's insecure-credential guard to refuse") + } + if !strings.Contains(err.Error(), "refusing to send a credential") { + t.Errorf("Handshake error = %v, want the insecure-credential refusal", err) + } + if len(server.seen) != 0 { + t.Errorf("server saw %d request(s); want the guard to refuse before ever dialing", len(server.seen)) + } + + snap, err := Handshake(context.Background(), target, DialOptions{HTTPClient: redirecting, AllowInsecureCredential: true}) + if err != nil { + t.Fatalf("Handshake with AllowInsecureCredential=true: %v", err) + } + if snap.ProjectId != "proj-insecure-wiring" { + t.Errorf("project_id = %q, want proj-insecure-wiring", snap.ProjectId) + } +} + +// TestDialWithWiresTargetAllowInsecureCredentialThroughToTheGuard is +// TestDialWithWiresAllowInsecureCredentialThroughToTheGuard's companion for +// the PERSISTED grant (bee-ghosttrack CHANGES_REQUESTED on #7288, +// should-fix 2): `bd connect --allow-plaintext` now saves the opt-in onto +// the sidecar (Target.AllowInsecureCredential, round-tripped through +// SaveTarget/LoadTarget), so an ORDINARY command's dial — which builds +// DialOptions fresh with AllowInsecureCredential left false, unlike +// connect's own Handshake probe — must still get through on target's say +// alone. Without DialWith also consulting target.AllowInsecureCredential, a +// workspace connected with the flag would need BEADS_HTTP_ALLOW_INSECURE=1 +// set for every later command too, which is exactly the gap this closes. +func TestDialWithWiresTargetAllowInsecureCredentialThroughToTheGuard(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, nonLoopbackTestHost+"=integration-token") + server := &contextServer{body: v0Context("proj-insecure-wiring-target")} + srv := server.start(t) + + redirecting := &http.Client{Transport: &redirectToAddrTransport{addr: strings.TrimPrefix(srv.URL, "http://")}} + + plainTarget := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + if _, err := Handshake(context.Background(), plainTarget, DialOptions{HTTPClient: redirecting}); err == nil { + t.Fatal("Handshake with no grant anywhere succeeded; want the guard to refuse") + } + + grantedTarget := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/"), AllowInsecureCredential: true} + snap, err := Handshake(context.Background(), grantedTarget, DialOptions{HTTPClient: redirecting}) + if err != nil { + t.Fatalf("Handshake with target.AllowInsecureCredential=true (opts left false): %v", err) + } + if snap.ProjectId != "proj-insecure-wiring-target" { + t.Errorf("project_id = %q, want proj-insecure-wiring-target", snap.ProjectId) + } +} + +// redirectToAddrTransport sends every request to addr instead of req.URL's +// own host, so a test can dial a real (loopback) httptest.Server while +// target.BaseURL carries a different, non-loopback-looking host — the shape +// needed to exercise the insecure-credential guard against a server that +// actually answers once the guard lets a request through. +type redirectToAddrTransport struct{ addr string } + +func (r *redirectToAddrTransport) RoundTrip(req *http.Request) (*http.Response, error) { + clone := req.Clone(req.Context()) + clone.URL.Host = r.addr + clone.Host = r.addr + return http.DefaultTransport.RoundTrip(clone) +} + +// noopProvider never attaches anything — the ambient ladder's unconfigured +// state, or a third-party provider that only sometimes authorizes. +type noopProvider struct{} + +func (noopProvider) Authorize(context.Context, *http.Request) error { return nil } +func (noopProvider) Refresh(context.Context) (bool, error) { return false, nil } + +// reportingProvider is a minimal CredentialProvider that also implements the +// wire package's unexported credentialSourceReporter shape structurally, to +// prove insecureCredentialGuard forwards both Refresh and Source. +type reportingProvider struct{ source string } + +func (p *reportingProvider) Authorize(context.Context, *http.Request) error { return nil } +func (p *reportingProvider) Refresh(context.Context) (bool, error) { return true, nil } +func (p *reportingProvider) Source() string { return p.source } diff --git a/internal/httpclient/metadata.go b/internal/httpclient/metadata.go index a58c5619c..fe0bc207e 100644 --- a/internal/httpclient/metadata.go +++ b/internal/httpclient/metadata.go @@ -112,7 +112,10 @@ func (s *Store) SetLocalMetadata(_ context.Context, key, value string) error { // A wrong-server handshake is the one error that must NOT collapse to that // benign skip (ga-b8ddd.11, Option C): the mismatch carries the id the server // actually owns, so handing that Got value back is what lets cmd/bd's -// validateWorkspaceIdentity compare it against metadata.json and FIRE the +// validateWorkspaceIdentity compare it against the workspace's expected +// project id — the sidecar's ExpectProjectID for this backend, not +// metadata.json (bee-ghosttrack CHANGES_REQUESTED on #7288: metadata.json's +// project_id is not kept current across `bd connect`) — and FIRE the // identity check — the write is refused, exit 1. What this path does NOT surface // is the wire ProjectMismatchError's own text: only the Got id survives this // method, so the server URL, database, repo root and recovery clause it carries diff --git a/internal/httpclient/open_with.go b/internal/httpclient/open_with.go new file mode 100644 index 000000000..22f6e9024 --- /dev/null +++ b/internal/httpclient/open_with.go @@ -0,0 +1,69 @@ +// Written fresh for OSS beads S6 (no bd-enterprise source copied). A sibling +// enterprise implementation takes over credential selection with its own +// Options.Credentials hook; OSS instead threads the backends.OpenOptions.Credential +// seam S1 defined, which backend_credential.go's ResolveCredential (S2) was +// built to resolve but which, until this file, had no production caller. +// OpenWith is that caller. +package httpclient + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/backends" +) + +// OpenWith is the backends.Backend.OpenWith hook backend/http (bdhttp) +// installs: the per-open seam that honors opts.Credential, opts.HTTPClient and +// opts.UserAgent for exactly THIS open, rather than the process-wide dialer +// Open/OpenReadOnly (NewFromConfig/NewReadOnlyFromConfig) use via the +// package-level registered dialer. +// +// That distinction is the whole reason this is a separate code path: the +// registered dialer is installed once, at process start, from one fixed +// DialOptions — exactly what a single-tenant CLI process wants, but useless +// to a multi-tenant embedder (gc, Gas City) that needs a different credential +// per workspace it opens in the same process. OpenWith dials fresh every call. +// +// base supplies the Register-time defaults (bdhttp.Options.UserAgent / +// HTTPClient) that a non-empty opts.UserAgent or non-nil opts.HTTPClient +// overrides for this one open. +// +// require is bdhttp.Options.RequireCredential: when true, a nil opts.Credential +// is ErrCredentialRequired rather than a silent fall-through to the ambient +// bearer ladder (BEADS_HTTP_TOKEN, BEADS_HTTP_TOKEN_COMMAND, the credentials +// file) — process-global ambient state a multi-tenant embedder cannot trust to +// name the right tenant. See ResolveCredential. +// +// Unlike Open/OpenReadOnly, OpenWith never consults BEADS_HTTP_CA_FILE: CA +// resolution is pinned to target.CAFile alone via DialOptionsForTarget, +// because an embedder juggling several targets in one process must not have +// one silently renarrowed (or widened) by an ambient value an unrelated open +// exported. See DialOptionsForTarget's own doc for why that rung is skipped +// here specifically. +func OpenWith(ctx context.Context, beadsDir string, opts backends.OpenOptions, base DialOptions, require bool) (storage.DoltStorage, error) { + target, err := LoadTarget(beadsDir) + if err != nil { + return nil, err + } + creds, err := ResolveCredential(opts, target.BaseURL, require) + if err != nil { + return nil, err + } + dialOpts := base + if opts.UserAgent != "" { + dialOpts.UserAgent = opts.UserAgent + } + if opts.HTTPClient != nil { + dialOpts.HTTPClient = opts.HTTPClient + } + dialOpts = DialOptionsForTarget(target, dialOpts) + conn, err := DialWith(target, creds, dialOpts) + if err != nil { + return nil, fmt.Errorf("dialing %s: %w", target, err) + } + s := New(target, conn, nil) + s.local = newLocalMetadata(beadsDir) + return s, nil +} diff --git a/internal/httpclient/open_with_test.go b/internal/httpclient/open_with_test.go new file mode 100644 index 000000000..ed23ed473 --- /dev/null +++ b/internal/httpclient/open_with_test.go @@ -0,0 +1,67 @@ +// Written fresh for OSS beads S6 (no bd-enterprise source copied). +package httpclient + +import ( + "context" + "net/http" + "net/url" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/backends" +) + +// literalSecretProvider is a CredentialProvider whose unexported field holds +// the literal secret value, the same shape a real token-bearing provider +// takes. It exists only so a test can grep for that literal in an error +// string: fmt's %+v prints unexported struct fields by value, so if OpenWith +// (or anything it calls) ever formats opts.Credential directly into an error +// — rather than formatting only the target and the wrapped dial error, as +// open_with.go's "dialing %s: %w" does today — this field's value leaks into +// logs and terminals, exactly what wire.CredentialProvider's own doc comment +// on Authorize forbids ("The error must not carry the credential"). +type literalSecretProvider struct { + secret string +} + +func (literalSecretProvider) Authorize(context.Context, *http.Request) error { return nil } +func (literalSecretProvider) Refresh(context.Context) (bool, error) { return false, nil } + +// TestOpenWithDialErrorNeverFormatsTheCredential kills the mutation that +// widens open_with.go's dial-failure wrap from +// `fmt.Errorf("dialing %s: %w", target, err)` to one that also interpolates +// opts.Credential (e.g. "dialing %s cred=%+v: %w"). A credential a caller +// supplies through OpenOptions.Credential must never surface in an error this +// package hands back — the error travels into the embedder's own logs and +// terminals, same as the doc comment on CredentialProvider.Authorize says. +// +// The dial target names a CA file that does not exist, which DialWith (via +// dialTransport) refuses before opening any socket: a config-time error, not +// a network one, so the test stays fast and never reaches any real network +// — the standing safety rule for this suite — while still giving OpenWith a +// real, non-nil error from the exact call it wraps. +func TestOpenWithDialErrorNeverFormatsTheCredential(t *testing.T) { + beadsDir := t.TempDir() + base, err := url.Parse("http://127.0.0.1:1/") + if err != nil { + t.Fatalf("parse base url: %v", err) + } + target := Target{BaseURL: base, CAFile: filepath.Join(beadsDir, "no-such-ca.pem")} + if err := SaveTarget(beadsDir, target); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + + const secret = "KILL-ME-SECRET-1234" + opts := backends.OpenOptions{ + Credential: ProvidedCredential{Provider: literalSecretProvider{secret: secret}}, + } + + _, err = OpenWith(context.Background(), beadsDir, opts, DialOptions{}, false) + if err == nil { + t.Fatal("OpenWith against an unreachable loopback port returned no error; cannot assert what its error does, or does not, carry") + } + if strings.Contains(err.Error(), secret) { + t.Errorf("OpenWith's error carries the literal credential secret %q: %v", secret, err) + } +} diff --git a/internal/httpclient/roles_graph.go b/internal/httpclient/roles_graph.go index 4cd3a85e5..41f831959 100644 --- a/internal/httpclient/roles_graph.go +++ b/internal/httpclient/roles_graph.go @@ -274,6 +274,106 @@ func (s *Store) GetDependencyRecords(ctx context.Context, id string) ([]*types.D // rather than chunking past it. See httpGraphCounter.CountEdges. const maxEdgeCountAnchors = 100 +// GetAllDependencyRecords answers storage.DoltStorage's broad, unfiltered +// dependency dump: every stored edge, grouped by its owning issue's id — the +// same shape issueops.GetAllDependencyRecordsInTx returns for a local +// backend. +// +// It overrides the unsupported_gen.go stub (design 3.6, HIGH-1): the +// external-deps decorator's loadBlockingState falls back to this method +// whenever the inner store implements neither +// storage.ExternalDependencyPolicyProber-reported server enforcement nor the +// narrower storage.ExternalDependencyQueryStore, which is exactly this +// backend's shape against a server that does NOT advertise +// wire.CapExternalDependencies. Refusing here (the old unsupported behavior) +// would make `bd ready`/`bd close`/claim hard-fail against such a server +// instead of correctly enforcing the policy client-side — the regression +// HIGH-1 reports. +// +// There is no single wire operation for "every dependency in the workspace": +// listDependencies is anchored at up to maxEdgeCountAnchors ids per call, so +// this walks every issue id over the full-exhaustion list (allIssueIDs) and +// reads its edges back in chunks of that size. It is O(issues/100) requests, +// which only this fallback path ever pays — a server that advertises the +// capability, or that implements the narrower query, never reaches this +// method. +func (s *Store) GetAllDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + ids, err := s.allIssueIDs(ctx) + if err != nil { + return nil, fmt.Errorf("list all issue ids: %w", err) + } + result := make(map[string][]*types.Dependency) + for _, chunk := range chunkStrings(ids, maxEdgeCountAnchors) { + q := url.Values{} + for _, id := range chunk { + q.Add("issue_id", id) + } + var body apigen.DependencyEdges + if err := s.dispatch(ctx, wire.Request{ + Op: wire.OpListDependencies, + Method: http.MethodGet, + Path: wire.PathDependencies, + Query: q, + }, &body); err != nil { + return nil, err + } + for i := range body.Items { + edge := &body.Items[i] + result[edge.IssueID] = append(result[edge.IssueID], edge) + } + } + return result, nil +} + +// allIssueIDs enumerates every issue id in the workspace, regardless of +// status or kind: closed, templates, gates, infra and ephemeral/wisp issues +// are all included (the list operation's narrower defaults otherwise exclude +// several of those), because a dependency row can be owned by any of them and +// GetAllDependencyRecords promises the whole table, not the default `bd +// list` view of it. +// +// want=0, maxRows=0 walks fetchIssuePages to full exhaustion rather than one +// page. +func (s *Store) allIssueIDs(ctx context.Context) ([]string, error) { + q := url.Values{ + "all": {"true"}, + "include_templates": {"true"}, + "include_gates": {"true"}, + "include_infra": {"true"}, + "include_ephemeral": {"true"}, + } + rows, err := s.fetchIssuePages(ctx, q, true, 0, 0, "", nil) + if err != nil { + return nil, err + } + ids := make([]string, 0, len(rows)) + for _, row := range rows { + if row == nil || row.Issue == nil || row.ID == "" { + continue + } + ids = append(ids, row.ID) + } + return ids, nil +} + +// chunkStrings splits ids into slices of at most size, preserving order. size +// must be positive; the last chunk may be shorter. +func chunkStrings(ids []string, size int) [][]string { + if len(ids) == 0 { + return nil + } + chunks := make([][]string, 0, (len(ids)+size-1)/size) + for len(ids) > 0 { + n := size + if n > len(ids) { + n = len(ids) + } + chunks = append(chunks, ids[:n]) + ids = ids[n:] + } + return chunks +} + // httpGraphCounter serves CountEdges from countDependencyEdges — the // TWENTY-SECOND wire-backed accessor, and the only role on this surface whose // answer is a number PER ANCHOR. diff --git a/internal/httpclient/served_external_dependency_policy_test.go b/internal/httpclient/served_external_dependency_policy_test.go new file mode 100644 index 000000000..92e0e9480 --- /dev/null +++ b/internal/httpclient/served_external_dependency_policy_test.go @@ -0,0 +1,492 @@ +//go:build cgo + +package httpclient + +import ( + "errors" + "fmt" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/externaldeps" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// TestServedReadyExcludesUnsatisfiedExternalDependencyWithoutServerCapability +// reproduces the S6 review's HIGH-1 regression: cmd/bd's +// wireExternalDependencyPolicy used to skip wrapping ANY registered remote +// backend outright (storage.RemoteBackendStore), so an issue blocked by an +// external:: dependency was listed by `bd ready` over +// http even though the identical workspace against a local backend would +// exclude it — the policy was silently skipped, not deferred to a server that +// claimed to enforce it (design 3.6). +// +// This pins the fix at the layer design 3.6 actually gates on: the decorator +// must still apply client-side enforcement whenever the server's handshake +// does not advertise wire.CapExternalDependencies. Today's OSS httpapi never +// advertises it (confirmed below), so this also exercises the http backend's +// GetAllDependencyRecords fallback (loadBlockingState's compatibility path) +// against a real served server, end to end. +func TestServedReadyExcludesUnsatisfiedExternalDependencyWithoutServerCapability(t *testing.T) { + env := newServedEnv(t, "hixd") + ctx := t.Context() + + blockedID := env.prefix + "-blocked" + freeID := env.prefix + "-free" + if err := env.createIssue(ctx, &types.Issue{ + ID: blockedID, Title: "blocked by an external capability", + Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "tester"); err != nil { + t.Fatalf("seed blocked issue: %v", err) + } + if err := env.createIssue(ctx, &types.Issue{ + ID: freeID, Title: "not blocked by anything", + Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "tester"); err != nil { + t.Fatalf("seed free issue: %v", err) + } + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: blockedID, + DependsOnID: "external:otherproj:cap", + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + + // The premise this test pins: the server this harness serves does not + // advertise wire.CapExternalDependencies. If a future change makes OSS + // httpapi advertise it, this test's exclusion no longer proves the + // client-side path and must be re-pointed at a server that genuinely + // withholds the capability. + if enforced, err := env.subject.ServerEnforcesExternalDependencyPolicy(ctx); err != nil { + t.Fatalf("ServerEnforcesExternalDependencyPolicy: %v", err) + } else if enforced { + t.Fatal("served test server unexpectedly advertises wire.CapExternalDependencies; this test's no-server-enforcement premise no longer holds") + } + + // externaldeps.New with nil locate/open funcs is the same fail-closed + // shape cmd/bd's wireExternalDependencyPolicy composes for an + // unconfigured external project: a reference to a project this workspace + // has not configured resolves to "unsatisfied", so the dependency keeps + // blocking rather than silently passing. + decorated := externaldeps.New(env.subject, nil, nil) + + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{}) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + rows, err := decorated.GetReadyWorkWithCounts(ctx, filter) + if err != nil { + t.Fatalf("GetReadyWorkWithCounts over http: %v", err) + } + var ids []string + for _, row := range rows { + if row != nil && row.Issue != nil { + ids = append(ids, row.ID) + } + } + for _, id := range ids { + if id == blockedID { + t.Errorf("ready over http = %v, want %q excluded by its unsatisfied external:otherproj:cap dependency", ids, blockedID) + } + } + var sawFree bool + for _, id := range ids { + if id == freeID { + sawFree = true + } + } + if !sawFree { + t.Errorf("ready over http = %v, want unrelated issue %q present", ids, freeID) + } + + // The total `bd ready --json` prints beside the page, and the count + // text-mode `bd ready` prints, must exclude the same issue. This store + // cannot express ExcludeIDs, so both come from the decorator's client-side + // drop, which is where the exclusion has to be subtracted. + _, total, err := decorated.GetReadyWorkWithCountsAndTotal(ctx, filter) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal over http: %v", err) + } + count, err := decorated.CountReadyWork(ctx, filter) + if err != nil { + t.Fatalf("CountReadyWork over http: %v", err) + } + if total != len(ids) || count != len(ids) { + t.Errorf("ready over http lists %v, but total = %d and CountReadyWork = %d; want both %d (%q excluded)", ids, total, count, len(ids), blockedID) + } +} + +// TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage pins, +// against a real served server, the shape the decorator once misread: an +// unlimited ready listing over http comes back as the server's default page +// (the wire omits a zero limit) while its total counts the whole ready set. +// Text-mode `bd ready` counts through CountReadyWork, which always fetches +// unlimited, so an externally blocked issue sorted past that page stayed in +// its count when the decorator took the zero limit for a window holding every +// row. +func TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage(t *testing.T) { + env := newServedEnv(t, "hixp") + ctx := t.Context() + + free := make([]*types.Issue, 0, workapi.DefaultReadyLimit+1) + for i := range workapi.DefaultReadyLimit + 1 { + free = append(free, &types.Issue{ + ID: fmt.Sprintf("%s-free%03d", env.prefix, i), Title: "not blocked by anything", + Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }) + } + if err := env.reference.CreateIssues(ctx, free, "tester"); err != nil { + t.Fatalf("seed free issues: %v", err) + } + // Priority 4 sorts the blocked issue after every free one, past the + // default page. + blockedID := env.prefix + "-blocked" + if err := env.createIssue(ctx, &types.Issue{ + ID: blockedID, Title: "blocked by an external capability", + Status: types.StatusOpen, Priority: 4, IssueType: types.TypeTask, + }, "tester"); err != nil { + t.Fatalf("seed blocked issue: %v", err) + } + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: blockedID, + DependsOnID: "external:otherproj:cap", + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + + countFilter, err := workapi.BuildReadyCountFilter(issueops.ReadyRequest{}) + if err != nil { + t.Fatalf("BuildReadyCountFilter: %v", err) + } + // The premise this test pins: undecorated, the unlimited listing is the + // default page, the blocked issue is not on it, and the total counts it. + rows, total, err := env.subject.GetReadyWorkWithCountsAndTotal(ctx, countFilter) + if err != nil { + t.Fatalf("undecorated GetReadyWorkWithCountsAndTotal over http: %v", err) + } + if len(rows) != workapi.DefaultReadyLimit || total != workapi.DefaultReadyLimit+2 { + t.Fatalf("undecorated unlimited listing over http = %d rows, total %d; want the %d-row default page, total %d; this test's premise no longer holds", + len(rows), total, workapi.DefaultReadyLimit, workapi.DefaultReadyLimit+2) + } + for _, row := range rows { + if row != nil && row.Issue != nil && row.ID == blockedID { + t.Fatalf("%q is on the default page; this test needs it past the page", blockedID) + } + } + + decorated := externaldeps.New(env.subject, nil, nil) + const want = workapi.DefaultReadyLimit + 1 // every free issue + count, err := decorated.CountReadyWork(ctx, countFilter) + if err != nil { + t.Fatalf("CountReadyWork over http: %v", err) + } + pageFilter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{}) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + pageRows, pageTotal, err := decorated.GetReadyWorkWithCountsAndTotal(ctx, pageFilter) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal over http: %v", err) + } + if count != want || pageTotal != want || len(pageRows) != workapi.DefaultReadyLimit { + t.Errorf("over http: CountReadyWork = %d; default page = %d rows, total %d; want count and total %d (%q excluded), %d rows", + count, len(pageRows), pageTotal, want, blockedID, workapi.DefaultReadyLimit) + } +} + +// TestServedRolesKeepTheExternalDependencyPolicy pins, against a real served +// server, the roles the decorator composes over a remote store. Each one +// answers through httpclient.Store's served roles: the base accessors built +// them over legacy methods that store refuses, which failed `bd list`, +// `bd show`, `bd dep tree`, `bd ready --claim` and `bd close` even for an issue +// nothing blocks. And each still holds the externally blocked issue out of +// ready work, claims and closes. The served handler here binds its roles to +// the raw reference store, which knows nothing of the policy, so every +// exclusion below is the client's. +func TestServedRolesKeepTheExternalDependencyPolicy(t *testing.T) { + env := newServedEnv(t, "hixr") + ctx := t.Context() + + const externalRef = "external:otherproj:cap" + blockedID := env.prefix + "-blocked" + freeID := env.prefix + "-free" + // Priority 1 sorts the blocked issue first, where a claim blind to the + // policy would take it. + for _, seed := range []*types.Issue{ + {ID: blockedID, Title: "blocked by an external capability", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeTask}, + {ID: freeID, Title: "not blocked by anything", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + } { + if err := env.createIssue(ctx, seed, "tester"); err != nil { + t.Fatalf("seed %s: %v", seed.ID, err) + } + } + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: blockedID, + DependsOnID: externalRef, + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + decorated := externaldeps.New(env.subject, nil, nil) + + reader, err := decorated.IssueReader() + if err != nil { + t.Fatalf("IssueReader: %v", err) + } + listed, err := reader.List(ctx, issueops.ListRequest{}) + if err != nil { + t.Fatalf("List over http: %v", err) + } + if ids := pageIDs(listed); !slices.Contains(ids, blockedID) || !slices.Contains(ids, freeID) { + t.Errorf("List over http = %v, want both issues: a plain listing claims nothing about readiness", ids) + } + readyListed, err := reader.List(ctx, issueops.ListRequest{ReadyFlag: true}) + if err != nil { + t.Fatalf("List --ready over http: %v", err) + } + if ids := pageIDs(readyListed); !slices.Equal(ids, []string{freeID}) { + t.Errorf("List --ready over http = %v, want [%s]", ids, freeID) + } + ready, err := reader.Ready(ctx, issueops.ReadyRequest{}) + if err != nil { + t.Fatalf("Ready over http: %v", err) + } + if ids := pageIDs(ready); !slices.Equal(ids, []string{freeID}) { + t.Errorf("Ready over http = %v, want [%s]", ids, freeID) + } + details, err := reader.Get(ctx, issueops.GetRequest{ID: blockedID}) + if err != nil || details == nil || details.ID != blockedID { + t.Fatalf("Get %s over http = %v, %v", blockedID, details, err) + } + + annotator, err := decorated.BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator: %v", err) + } + blocking, err := annotator.AnnotateBlocking(ctx, issueops.BlockingRequest{IDs: []string{blockedID, freeID}}) + if err != nil { + t.Fatalf("AnnotateBlocking over http: %v", err) + } + for _, item := range blocking.Items { + if blockedBy := slices.Contains(item.BlockedBy, externalRef); blockedBy != (item.ID == blockedID) { + t.Errorf("AnnotateBlocking over http: %s blocked by %v", item.ID, item.BlockedBy) + } + } + + walker, err := decorated.TreeWalker() + if err != nil { + t.Fatalf("TreeWalker: %v", err) + } + tree, err := walker.WalkTree(ctx, issueops.WalkTreeRequest{RootID: blockedID, MaxDepth: 50}) + if err != nil { + t.Fatalf("WalkTree over http: %v", err) + } + var leaves int + for _, node := range tree.Nodes { + if node.ID == externalRef { + leaves++ + } + } + if leaves != 1 { + t.Errorf("WalkTree over http shows %d %s leaves, want one", leaves, externalRef) + } + + claimer, err := decorated.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer: %v", err) + } + if _, err := claimer.Claim(ctx, issueops.ClaimRequest{Actor: "worker", IssueID: blockedID}); !errors.Is(err, storage.ErrCloseBlocked) { + t.Errorf("Claim %s over http = %v, want ErrCloseBlocked", blockedID, err) + } + readyClaimer, err := decorated.ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer: %v", err) + } + next, err := readyClaimer.ClaimNext(ctx, issueops.ClaimNextRequest{Actor: "worker"}) + if err != nil || next.Claimed == nil || next.Claimed.ID != freeID { + t.Fatalf("ClaimNext over http = %+v, %v; want %s", next.Claimed, err, freeID) + } + + closer, err := decorated.BatchCloser() + if err != nil { + t.Fatalf("BatchCloser: %v", err) + } + closed, err := closer.CloseBatch(ctx, issueops.CloseBatchRequest{ + Actor: "worker", + Items: []issueops.BatchCloseItem{{IssueID: blockedID}, {IssueID: freeID}}, + }) + if err != nil || len(closed.Outcomes) != 2 { + t.Fatalf("CloseBatch over http = %+v, %v; want two outcomes", closed.Outcomes, err) + } + if !errors.Is(closed.Outcomes[0].Err, storage.ErrCloseBlocked) || closed.Outcomes[1].Err != nil { + t.Fatalf("CloseBatch over http: %s = %v, %s = %v; want the first refused, the second closed", + blockedID, closed.Outcomes[0].Err, freeID, closed.Outcomes[1].Err) + } + assertServedStatus(t, env, blockedID, types.StatusOpen) + assertServedStatus(t, env, freeID, types.StatusClosed) + forced, err := closer.CloseBatch(ctx, issueops.CloseBatchRequest{ + Actor: "worker", Force: true, + Items: []issueops.BatchCloseItem{{IssueID: blockedID}}, + }) + if err != nil || len(forced.Outcomes) != 1 || forced.Outcomes[0].Err != nil { + t.Fatalf("CloseBatch --force over http = %+v, %v; want it closed", forced.Outcomes, err) + } + assertServedStatus(t, env, blockedID, types.StatusClosed) +} + +// TestServedReadyTotalIgnoresAnExternallyBlockedIssueOutsideTheFilter pins the +// count an assignee-filtered `bd ready` prints over http. Past the server's +// default page the decorator used to subtract every externally blocked issue +// in the workspace from the filtered total, so bob's blocked issue, which the +// filter never counted, took one off alice's. +func TestServedReadyTotalIgnoresAnExternallyBlockedIssueOutsideTheFilter(t *testing.T) { + env := newServedEnv(t, "hixa") + ctx := t.Context() + + mine := make([]*types.Issue, 0, workapi.DefaultReadyLimit+1) + for i := range workapi.DefaultReadyLimit + 1 { + mine = append(mine, &types.Issue{ + ID: fmt.Sprintf("%s-alice%03d", env.prefix, i), Title: "alice's ready work", + Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, Assignee: "alice", + }) + } + if err := env.reference.CreateIssues(ctx, mine, "tester"); err != nil { + t.Fatalf("seed alice's issues: %v", err) + } + blockedID := env.prefix + "-bob" + if err := env.createIssue(ctx, &types.Issue{ + ID: blockedID, Title: "bob's externally blocked work", + Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, Assignee: "bob", + }, "tester"); err != nil { + t.Fatalf("seed blocked issue: %v", err) + } + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: blockedID, + DependsOnID: "external:otherproj:cap", + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + + decorated := externaldeps.New(env.subject, nil, nil) + const want = workapi.DefaultReadyLimit + 1 // every one of alice's issues + request := issueops.ReadyRequest{Assignee: "alice"} + countFilter, err := workapi.BuildReadyCountFilter(request) + if err != nil { + t.Fatalf("BuildReadyCountFilter: %v", err) + } + count, err := decorated.CountReadyWork(ctx, countFilter) + if err != nil { + t.Fatalf("CountReadyWork over http: %v", err) + } + pageFilter, err := workapi.BuildReadyFilter(request) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + rows, total, err := decorated.GetReadyWorkWithCountsAndTotal(ctx, pageFilter) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal over http: %v", err) + } + if count != want || total != want || len(rows) != workapi.DefaultReadyLimit { + t.Errorf("over http, alice's ready work: CountReadyWork = %d; page = %d rows, total %d; want count and total %d, %d rows", + count, len(rows), total, want, workapi.DefaultReadyLimit) + } +} + +// TestServedReadyMaxRowsCountsTheRowsKept pins `bd ready --limit N +// --max-rows N` over http, which failed with exit 2 once more than N issues +// were ready and any issue held an unsatisfied external blocker. The decorator +// fetches a window wider than the limit to make room for the rows it drops, +// and the http bridge caps the rows it fetched, so the cap counted rows the +// caller was never handed. A SQL store caps the page it delivers, and so must +// this path. +func TestServedReadyMaxRowsCountsTheRowsKept(t *testing.T) { + env := newServedEnv(t, "hixm") + ctx := t.Context() + + free := []string{env.prefix + "-a", env.prefix + "-b", env.prefix + "-c"} + blockedID := env.prefix + "-z" + // The priorities fix the ready order: the three free issues, then the + // blocked one, so a window of the limit plus one can leave it out and + // still hold more rows than the limit. + for i, id := range append(slices.Clone(free), blockedID) { + if err := env.createIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: i + 1, IssueType: types.TypeTask, + }, "tester"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: blockedID, + DependsOnID: "external:otherproj:cap", + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + decorated := externaldeps.New(env.subject, nil, nil) + + for _, tc := range []struct { + name string + limit, maxRows int + want []string // nil: the cap fires + }{ + // The window is 2+1 rows, all free; the cap counts the page of two. + {name: "cap equals the limit", limit: 2, maxRows: 2, want: free[:2]}, + // The window holds all four rows, three of them kept. + {name: "cap below the limit", limit: 5, maxRows: 3, want: free}, + // The server's default page, again all four. + {name: "unlimited", limit: 0, maxRows: 3, want: free}, + // The three rows kept exceed the caller's cap, which still fires. + {name: "kept rows exceed the cap", limit: 5, maxRows: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + limit := tc.limit + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{Limit: &limit}) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + filter.MaxRows, filter.MaxRowsSource = tc.maxRows, "--max-rows" + check := func(method string, ids []string, err error) { + t.Helper() + var tooMany *storageops.ErrTooManyRows + switch { + case tc.want == nil: + if !errors.As(err, &tooMany) || tooMany.Cap != tc.maxRows || tooMany.Source != "--max-rows" { + t.Errorf("%s over http = %v, %v; want the --max-rows cap of %d to fire on the rows kept", method, ids, err, tc.maxRows) + } + case err != nil: + t.Errorf("%s over http: %v; want %v, which a cap of %d admits", method, err, tc.want, tc.maxRows) + case !slices.Equal(ids, tc.want): + t.Errorf("%s over http = %v, want %v (%q excluded)", method, ids, tc.want, blockedID) + } + } + + issues, err := decorated.GetReadyWork(ctx, filter) + check("GetReadyWork", issueIDs(issues), err) + rows, err := decorated.GetReadyWorkWithCounts(ctx, filter) + check("GetReadyWorkWithCounts", rowIDs(rows), err) + rows, total, err := decorated.GetReadyWorkWithCountsAndTotal(ctx, filter) + check("GetReadyWorkWithCountsAndTotal", rowIDs(rows), err) + if err == nil && total != len(free) { + t.Errorf("GetReadyWorkWithCountsAndTotal over http: total %d, want %d (%q excluded)", total, len(free), blockedID) + } + }) + } +} + +func assertServedStatus(t *testing.T, env *servedEnv, id string, want types.Status) { + t.Helper() + issue, err := env.getIssue(t.Context(), id) + if err != nil || issue == nil { + t.Fatalf("read %s back: %v, %v", id, issue, err) + } + if issue.Status != want { + t.Errorf("%s status = %s, want %s", id, issue.Status, want) + } +} diff --git a/internal/httpclient/store.go b/internal/httpclient/store.go index 2b36e9744..731c5fa7c 100644 --- a/internal/httpclient/store.go +++ b/internal/httpclient/store.go @@ -18,9 +18,11 @@ import ( "context" "errors" "fmt" + "slices" "sync" "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" "github.com/steveyegge/beads/internal/storage" ) @@ -73,6 +75,58 @@ type Store struct { // interfaces here would assert a maintenance contract no OSS caller checks. var _ storage.DoltStorage = (*Store)(nil) +// IsRemoteBackendStore satisfies storage.RemoteBackendStore. cmd/bd's storage +// chain wraps this store in the external-dependency policy decorator like any +// other. The decorator reads this marker only to build its roles on this +// store's served roles rather than on the legacy methods it refuses +// (internal/storage/externaldeps/remote_roles.go). It decides whether to stand +// down through ServerEnforcesExternalDependencyPolicy below, never through +// this marker. +func (s *Store) IsRemoteBackendStore() bool { return true } + +var _ storage.RemoteBackendStore = (*Store)(nil) + +// ServerEnforcesExternalDependencyPolicy satisfies +// storage.ExternalDependencyPolicyProber (design 3.6): it answers whether the +// server this store dials already enforces bd's external-dependency policy +// itself, by checking the handshake for wire.CapExternalDependencies. +// +// It propagates a handshake error rather than swallowing it to false, the +// same asymmetry servesBatchClose documents: this probe gates whether the +// external-deps DECORATOR runs its own enforcement at all, so an unresolved +// "does the server already enforce this" has to fail the caller, not silently +// resolve to "no, so enforce here" (which happens to be safe for THIS probe's +// specific true/false meaning but would still hide a transport failure behind +// a policy decision) or to "yes, so skip" (which would be the actual bug this +// interface exists to prevent — the policy must never be silently skipped). +func (s *Store) ServerEnforcesExternalDependencyPolicy(ctx context.Context) (bool, error) { + snap, err := s.snapshot(ctx) + if err != nil { + return false, err + } + if snap == nil { + return false, nil + } + return slices.Contains(snap.Capabilities, wire.CapExternalDependencies), nil +} + +var _ storage.ExternalDependencyPolicyProber = (*Store)(nil) + +// ExcludeIDsUnsupported satisfies storage.ExcludeIDsUnsupportedStore: the v0 +// wire's listReadyWork and countReadyWork operations publish no id-exclusion +// parameter, so encode.ReadyBridgeParams / ReadyBridgeCountParams refuse any +// types.WorkFilter carrying ExcludeIDs outright (D8/D9, L12) rather than +// silently widening the result. Reporting true here — unconditionally, since +// the gap is in the wire shape, not in any per-call state — lets the +// external-deps decorator fall back to filtering its OWN additional +// exclusions (unsatisfied external:: blockers) client +// side instead of folding them into the filter it hands this store, so design +// 3.6's policy still runs against a server that does not advertise +// wire.CapExternalDependencies rather than tripping this store's refusal. +func (s *Store) ExcludeIDsUnsupported() bool { return true } + +var _ storage.ExcludeIDsUnsupportedStore = (*Store)(nil) + // New builds the store around an already-resolved target and transport. // // snapshot pre-seeds the handshake context, or is nil to let D6 fetch it lazily diff --git a/internal/httpclient/target.go b/internal/httpclient/target.go index ca86c9e77..bad29f6d0 100644 --- a/internal/httpclient/target.go +++ b/internal/httpclient/target.go @@ -30,6 +30,26 @@ type Target struct { // skips the check. It is compared against ContextResponse.project_id at // handshake (D6). ExpectProjectID string + // PreviousBackend records the backend selection metadata.json carried + // right before THIS connect (bee-ghosttrack CHANGES_REQUESTED on #7288, + // should-fix 1): `bd connect --clear` reads it back to restore + // metadata.json's backend selection, so detaching from http returns the + // workspace to whatever it was attached to before, rather than leaving + // it pinned to "http" with no sidecar to dial. "" means there was + // nothing to restore (a workspace's first-ever connect, or a value + // Attach deliberately left alone — see Attach's own doc). + PreviousBackend string + // AllowInsecureCredential records `bd connect --allow-plaintext`'s grant + // for THIS target (bee-ghosttrack CHANGES_REQUESTED on #7288, + // should-fix 2): once connected with the flag, every later dial for this + // workspace — not only connect's own Handshake probe — carries the same + // opt-in, so an operator who accepted the risk once at connect time does + // not also need BEADS_HTTP_ALLOW_INSECURE=1 set for every ordinary `bd` + // command afterward. It is scoped to the sidecar it is saved beside: a + // `bd connect` to a DIFFERENT url without the flag writes a fresh + // sidecar with this false, so the grant never silently carries over to a + // server it was never given for. See guardInsecureCredential. + AllowInsecureCredential bool // CAFile names a PEM file that becomes the ENTIRE trusted root pool for // this target — not an addition to the system store — set by // `bd connect --ca-file `. "" leaves this target on the system @@ -81,6 +101,14 @@ type targetFile struct { // CAFile mirrors Target.CAFile; see that field's doc for the trust model // and TransportFor for the env-vs-sidecar precedence. CAFile string `json:"ca_file,omitempty"` + // PreviousBackend mirrors Target.PreviousBackend; see that field's doc. + PreviousBackend string `json:"previous_backend,omitempty"` + // AllowPlaintext mirrors Target.AllowInsecureCredential; see that field's + // doc. Named differently on the wire (matching the CLI flag's own + // spelling) than the Go field (matching DialOptions.AllowInsecureCredential), + // deliberately: this is the one record of what a human typed, and the + // JSON key should read that way on disk. + AllowPlaintext bool `json:"allow_plaintext,omitempty"` } // TargetPath is the sidecar's location for a workspace. @@ -122,7 +150,13 @@ func LoadTarget(beadsDir string) (Target, error) { if err := checkCAFileAbsolute(f.CAFile); err != nil { return Target{}, err } - return Target{BaseURL: u, ExpectProjectID: f.ExpectProjectID, CAFile: f.CAFile}, nil + return Target{ + BaseURL: u, + ExpectProjectID: f.ExpectProjectID, + CAFile: f.CAFile, + PreviousBackend: f.PreviousBackend, + AllowInsecureCredential: f.AllowPlaintext, + }, nil } // checkNoUserinfo refuses a url carrying userinfo ("user:secret@host"). @@ -155,7 +189,10 @@ func checkCAFileAbsolute(caFile string) error { // and the connect command share one encoder; the connect UX itself (gitignore // coverage, identity verification, conversion consent) is not here. It refuses // a url carrying userinfo and a relative CAFile before writing anything, with -// LoadTarget's own messages. +// LoadTarget's own messages. The write is atomic (temp file in the same +// directory, then rename): a reader racing this write — LoadTarget, or +// another process entirely — must never observe a truncated or partial +// sidecar. func SaveTarget(beadsDir string, t Target) error { if err := checkNoUserinfo(t.BaseURL); err != nil { return err @@ -163,7 +200,13 @@ func SaveTarget(beadsDir string, t Target) error { if err := checkCAFileAbsolute(t.CAFile); err != nil { return err } - f := targetFile{ExpectProjectID: t.ExpectProjectID, API: "v0", CAFile: t.CAFile} + f := targetFile{ + ExpectProjectID: t.ExpectProjectID, + API: "v0", + CAFile: t.CAFile, + PreviousBackend: t.PreviousBackend, + AllowPlaintext: t.AllowInsecureCredential, + } if t.BaseURL != nil { f.URL = t.BaseURL.String() } @@ -171,12 +214,38 @@ func SaveTarget(beadsDir string, t Target) error { if err != nil { return fmt.Errorf("marshaling %s: %w", TargetFileName, err) } - if err := os.WriteFile(TargetPath(beadsDir), data, 0o600); err != nil { + if err := writeFileAtomic(TargetPath(beadsDir), data, 0o600); err != nil { return fmt.Errorf("writing %s: %w", TargetFileName, err) } return nil } +// writeFileAtomic writes data to a temp file in path's directory and renames +// it over path, so a concurrent reader never sees a truncated or partial +// file. Mirrors internal/configfile's own helper of the same name and shape; +// not shared directly because that one is package-private to configfile. +func writeFileAtomic(path string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, filepath.Base(path)+".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) //nolint:errcheck // no-op after successful rename + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Chmod(perm); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpName, path) +} + // RemoveTarget deletes the activation sidecar, reporting whether one was there. // It is `bd connect --clear`'s whole write: detaching is a per-user act, so it // must never touch the tracked metadata.json that selects the backend. diff --git a/internal/httpclient/target_test.go b/internal/httpclient/target_test.go index 39a915a50..e25ff8c6a 100644 --- a/internal/httpclient/target_test.go +++ b/internal/httpclient/target_test.go @@ -193,3 +193,64 @@ func TestLoadTargetAcceptsAbsoluteCAFile(t *testing.T) { t.Errorf("CAFile = %q, want %q", got.CAFile, abs) } } + +// TestSaveTargetRoundTripsPreviousBackendAndAllowInsecureCredential pins the +// bee-ghosttrack CHANGES_REQUESTED findings on #7288 (should-fix 1 and 2): +// `bd connect --clear` needs PreviousBackend back to restore metadata.json's +// backend selection, and an ordinary command's dial needs +// AllowInsecureCredential back to honor a plaintext grant made at connect +// time. Neither is useful if the sidecar does not actually carry it through +// a save/load round trip. +func TestSaveTargetRoundTripsPreviousBackendAndAllowInsecureCredential(t *testing.T) { + dir := t.TempDir() + target := Target{ + BaseURL: mustParseURL(t, "http://127.0.0.1:8080"), + PreviousBackend: "dolt", + AllowInsecureCredential: true, + } + if err := SaveTarget(dir, target); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + got, err := LoadTarget(dir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.PreviousBackend != "dolt" { + t.Errorf("PreviousBackend = %q, want %q", got.PreviousBackend, "dolt") + } + if !got.AllowInsecureCredential { + t.Error("AllowInsecureCredential = false after a round trip, want true") + } + + raw, err := os.ReadFile(TargetPath(dir)) + if err != nil { + t.Fatalf("reading sidecar: %v", err) + } + if !strings.Contains(string(raw), `"previous_backend": "dolt"`) { + t.Errorf("sidecar does not carry previous_backend: %s", raw) + } + if !strings.Contains(string(raw), `"allow_plaintext": true`) { + t.Errorf("sidecar does not carry allow_plaintext: %s", raw) + } +} + +// TestSaveTargetOmitsPreviousBackendAndAllowInsecureCredentialWhenUnset keeps +// the common case's sidecar exactly as small as before this fix: a workspace +// with nothing to restore and no plaintext grant must not gain two new +// always-present keys. +func TestSaveTargetOmitsPreviousBackendAndAllowInsecureCredentialWhenUnset(t *testing.T) { + dir := t.TempDir() + if err := SaveTarget(dir, testTarget(t)); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + raw, err := os.ReadFile(TargetPath(dir)) + if err != nil { + t.Fatalf("reading sidecar: %v", err) + } + if strings.Contains(string(raw), "previous_backend") { + t.Errorf("sidecar carries previous_backend with nothing to restore: %s", raw) + } + if strings.Contains(string(raw), "allow_plaintext") { + t.Errorf("sidecar carries allow_plaintext with no grant: %s", raw) + } +} diff --git a/internal/httpclient/unsupported_gen.go b/internal/httpclient/unsupported_gen.go index 2e8cde872..1299968ea 100644 --- a/internal/httpclient/unsupported_gen.go +++ b/internal/httpclient/unsupported_gen.go @@ -227,11 +227,6 @@ func (unsupportedDoltStorage) ForcePush(_ context.Context) (err error) { return } -func (unsupportedDoltStorage) GetAllDependencyRecords(_ context.Context) (_ map[string][]*types.Dependency, err error) { - err = errUnsupported("GetAllDependencyRecords") - return -} - func (unsupportedDoltStorage) GetAllEventsSince(_ context.Context, _ time.Time) (_ []*types.Event, err error) { err = errUnsupported("GetAllEventsSince") return @@ -742,4 +737,4 @@ func (unsupportedDoltStorage) VersionReconciler() (_ issueops.VersionReconciler, return } -// NOTE: partial shell (145 of 200 methods generated; 55 left to this package's hand-written set). +// NOTE: partial shell (144 of 200 methods generated; 56 left to this package's hand-written set). diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index b0ed49785..e29e8286a 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -240,18 +240,16 @@ const CapIssuesCreateDefaultPriority = "issues.create.defaultPriority" // through the policy layer, so httpapi.Capabilities() — the build-level list // the parity gate compares against — never contains it. // -// UNCONSUMED since S3 reconciliation (2026-10): THIS token is unread by this -// client, not because OSS lacks external-dependency policy enforcement — the -// externaldeps decorator (internal/storage/externaldeps) exists in OSS and is -// wired unconditionally into the local storage chain (cmd/bd/storage_chain.go), -// so a local backend already applies the policy itself. What this http -// client specifically lacks is a client-side COMPOSITION of that same -// decorator around a storage.ExternalDependencyQueryStore-backed remote -// store, because a remote server advertising this capability applies the -// policy on its own side before answering — there is nothing left for the -// client to decorate. The token stays declared, because the server-side -// capability and its wire spelling are real and S10 schedules the client half -// that reads it. +// Consumed since S6 (design 3.6): the externaldeps decorator +// (internal/storage/externaldeps) wraps the http store the same as any local +// backend (cmd/bd/storage_chain.go wires it unconditionally), and consults +// storage.ExternalDependencyPolicyProber — which httpclient.Store implements +// by checking for this token in the handshake — before deciding whether to +// also enforce the policy client-side. A server that advertises this token +// has already applied the policy itself, so the decorator's own check is +// redundant there. A server that does NOT advertise it gets the ordinary +// client-side enforcement: the policy is never silently skipped merely +// because the store is remote. const CapExternalDependencies = "policy.external_dependencies" // ClientWireRevision is the wire shape this client was built to speak and diff --git a/internal/storage/externaldeps/BUILD.bazel b/internal/storage/externaldeps/BUILD.bazel index 31715e16c..8ceaede23 100644 --- a/internal/storage/externaldeps/BUILD.bazel +++ b/internal/storage/externaldeps/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "batch_closer.go", "decorator.go", + "remote_roles.go", "resolver.go", "role_accessors.go", "uow_decorator.go", @@ -30,6 +31,7 @@ go_test( srcs = [ "decorator_test.go", "ready_wake_test.go", + "remote_roles_test.go", "uow_decorator_test.go", ], embed = [":externaldeps"], diff --git a/internal/storage/externaldeps/batch_closer.go b/internal/storage/externaldeps/batch_closer.go index 2a6b6970a..35ad20981 100644 --- a/internal/storage/externaldeps/batch_closer.go +++ b/internal/storage/externaldeps/batch_closer.go @@ -10,7 +10,12 @@ import ( ) // BatchCloser preserves external policy on bd close, including its next claim. +// Over a store that serves its roles, remoteBatchCloser refuses the next claim +// instead. func (s *Store) BatchCloser() (issueops.BatchCloser, error) { + if s.servesRoles() { + return &remoteBatchCloser{policy: s}, nil + } return &batchCloser{policy: s}, nil } diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go index 9edb03880..75642ecad 100644 --- a/internal/storage/externaldeps/decorator.go +++ b/internal/storage/externaldeps/decorator.go @@ -119,6 +119,19 @@ type blockingState struct { } func (s *Store) loadBlockingState(ctx context.Context) (blockingState, error) { + // Design 3.6: a remote server that already enforces this policy itself + // (advertised as the handshake's wire.CapExternalDependencies capability, + // probed here through storage.ExternalDependencyPolicyProber) has already + // applied it before answering, so this decorator's own pass would be + // redundant — skip to an empty blocking state. A store that does not + // implement the prober, or that implements it and reports false (a remote + // server silent on the capability included), falls through to the + // ordinary client-side enforcement below: the policy is never silently + // skipped merely because the inner store is remote. + if enforced, err := s.serverEnforcesPolicy(ctx); err != nil || enforced { + return blockingState{}, err + } + queryStore, ok := storage.UnwrapStore(s.inner).(storage.ExternalDependencyQueryStore) var allDeps map[string][]*types.Dependency var err error @@ -136,6 +149,20 @@ func (s *Store) loadBlockingState(ctx context.Context) (blockingState, error) { return s.blockingStateFromRecords(ctx, allDeps) } +// serverEnforcesPolicy asks the inner store's ExternalDependencyPolicyProber, +// if it has one, whether its server already enforced this policy. +func (s *Store) serverEnforcesPolicy(ctx context.Context) (bool, error) { + prober, ok := storage.UnwrapStore(s.inner).(storage.ExternalDependencyPolicyProber) + if !ok { + return false, nil + } + enforced, err := prober.ServerEnforcesExternalDependencyPolicy(ctx) + if err != nil { + return false, fmt.Errorf("external dependencies: probe server policy: %w", err) + } + return enforced, nil +} + func (s *Store) blockingStateFromRecords(ctx context.Context, allDeps map[string][]*types.Dependency) (blockingState, error) { refs := make([]reference, 0) refsByIssue := make(map[string][]string) @@ -176,8 +203,16 @@ func (s *Store) GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*t if err != nil { return nil, err } - filter = withExternalExclusions(filter, state.refsByIssue) - return s.inner.GetReadyWork(ctx, filter) + queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) + issues, err := s.inner.GetReadyWork(ctx, queryFilter) + if err != nil { + return nil, err + } + kept := dropBlockedIssues(issues, blockedIDs, filter.Limit) + if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { + return nil, err + } + return kept, nil } // GetReadyWorkWithCounts is the counts-bearing equivalent of GetReadyWork. @@ -186,8 +221,23 @@ func (s *Store) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFil if err != nil { return nil, err } - filter = withExternalExclusions(filter, state.refsByIssue) - return s.inner.GetReadyWorkWithCounts(ctx, filter) + return s.readyWorkWithCounts(ctx, filter, state) +} + +// readyWorkWithCounts is GetReadyWorkWithCounts' body against an +// already-loaded blockingState, which remoteReadyClaimer reuses across its +// passes. +func (s *Store) readyWorkWithCounts(ctx context.Context, filter types.WorkFilter, state blockingState) ([]*types.IssueWithCounts, error) { + queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) + rows, err := s.inner.GetReadyWorkWithCounts(ctx, queryFilter) + if err != nil { + return nil, err + } + kept, _ := dropBlockedIssuesWithCounts(rows, blockedIDs, filter.Limit) + if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { + return nil, err + } + return kept, nil } // GetReadyWorkWithCountsAndTotal applies the same external exclusions as @@ -199,8 +249,106 @@ func (s *Store) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types if err != nil { return nil, 0, err } - filter = withExternalExclusions(filter, state.refsByIssue) - return s.inner.GetReadyWorkWithCountsAndTotal(ctx, filter) + return s.readyWorkWithCountsAndTotal(ctx, filter, state) +} + +// readyExclusion resolves how this decorator applies its OWN additional +// exclusions (issues with an unsatisfied external blocker) against s.inner. +// +// A store that can express types.WorkFilter.ExcludeIDs over its own +// transport (every first-party SQL-backed store) gets them folded straight +// into the filter, exactly as before: the WHERE clause does the exclusion. +// +// A store that cannot (storage.ExcludeIDsUnsupportedStore — today only +// httpclient.Store: design 3.6 / L12 leaves listReadyWork with no +// id-exclusion parameter on the v0 wire at all) instead gets a bumped Limit +// and the exclusions applied client-side by the caller. Requesting +// Limit+len(refsByIssue) rows from the UNFILTERED ready set is provably +// enough headroom to still surface Limit genuinely-unblocked rows whenever +// that many exist: refsByIssue cannot hold more entries than the number of +// issues anywhere with an unsatisfied external blocker, so at most +// len(refsByIssue) of the bumped window's extra rows can be ones this +// decorator has to drop. Whichever path is taken, the policy itself is never +// skipped — only where the exclusion happens differs. +// +// The client-side path also takes the MaxRows cap off the inner query. That +// store caps the rows it fetched, which the bump lets outnumber Limit, so +// `--limit N --max-rows N` failed with exit 2 as soon as more than N issues +// were ready, refusing a page that could never exceed N. The caller enforces +// the cap on the rows it keeps instead (capKeptRows), after the Limit trim, +// which is where issueops enforces it on the rows a SQL store delivers +// (finishReadyWorkWithCounts). +func (s *Store) readyExclusion(filter types.WorkFilter, refsByIssue map[string][]string) (queryFilter types.WorkFilter, blockedIDs map[string]bool) { + if len(refsByIssue) == 0 { + return filter, nil + } + if unsupported, ok := storage.UnwrapStore(s.inner).(storage.ExcludeIDsUnsupportedStore); !ok || !unsupported.ExcludeIDsUnsupported() { + return withExternalExclusions(filter, refsByIssue), nil + } + blockedIDs = make(map[string]bool, len(refsByIssue)) + for issueID := range refsByIssue { + blockedIDs[issueID] = true + } + queryFilter = filter + if queryFilter.Limit > 0 { + queryFilter.Limit += len(blockedIDs) + } + queryFilter.MaxRows = 0 + queryFilter.MaxRowsSource = "" + return queryFilter, blockedIDs +} + +// capKeptRows enforces the MaxRows cap readyExclusion took off the inner +// query, on the kept rows the caller is handed. With nothing to drop +// client-side the inner query kept the cap and has already enforced it. +func capKeptRows(kept int, filter types.WorkFilter, blockedIDs map[string]bool) error { + if len(blockedIDs) == 0 { + return nil + } + return issueops.EnforceMaxRowsCap(kept, filter.MaxRows, filter.MaxRowsSource) +} + +// dropBlockedIssues is readyExclusion's client-side half: it drops the rows +// blockedIDs names and trims what remains to limit. It copies into a fresh +// slice rather than compacting issues in place, so the slice s.inner returned +// still holds every row it fetched. +func dropBlockedIssues(issues []*types.Issue, blockedIDs map[string]bool, limit int) []*types.Issue { + if len(blockedIDs) == 0 { + return issues + } + kept := make([]*types.Issue, 0, len(issues)) + for _, issue := range issues { + if issue != nil && blockedIDs[issue.ID] { + continue + } + kept = append(kept, issue) + } + if limit > 0 && len(kept) > limit { + kept = kept[:limit] + } + return kept +} + +// dropBlockedIssuesWithCounts is dropBlockedIssues for counts-bearing rows. It +// also reports how many rows it dropped from the whole fetched window (before +// the limit trim), which is the figure readyWorkWithCountsAndTotal subtracts +// from the inner store's total. +func dropBlockedIssuesWithCounts(rows []*types.IssueWithCounts, blockedIDs map[string]bool, limit int) (kept []*types.IssueWithCounts, dropped int) { + if len(blockedIDs) == 0 { + return rows, 0 + } + kept = make([]*types.IssueWithCounts, 0, len(rows)) + for _, row := range rows { + if row != nil && row.Issue != nil && blockedIDs[row.ID] { + dropped++ + continue + } + kept = append(kept, row) + } + if limit > 0 && len(kept) > limit { + kept = kept[:limit] + } + return kept, dropped } func withExternalExclusions(filter types.WorkFilter, refsByIssue map[string][]string) types.WorkFilter { @@ -216,6 +364,80 @@ func withExternalExclusions(filter types.WorkFilter, refsByIssue map[string][]st return filter } +// readyWorkWithCountsAndTotal is GetReadyWorkWithCountsAndTotal's body, split +// out so CountReadyWork's fallback (below) can reuse it against an +// already-loaded blockingState instead of loading it a second time. +func (s *Store) readyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter, state blockingState) ([]*types.IssueWithCounts, int, error) { + queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) + rows, total, err := s.inner.GetReadyWorkWithCountsAndTotal(ctx, queryFilter) + if err != nil { + return nil, 0, err + } + kept, blockedSeen := dropBlockedIssuesWithCounts(rows, blockedIDs, filter.Limit) + if len(blockedIDs) == 0 { + return kept, total, nil + } + if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { + return nil, 0, err + } + + var adjustedTotal int + if total <= len(rows) { + // The fetch's own reported total says the window already held the + // entire ready set, so every blocked row the window contains is every + // blocked row the ready set contains: the subtraction is exact. This + // is judged from the rows that came back, not from the Limit asked + // for: a zero Limit means every row to a SQL store, but the http wire + // omits it and the server answers with its default page + // (workapi.DefaultReadyLimit), so an unlimited fetch can come back + // truncated too. + adjustedTotal = total - blockedSeen + } else { + // The ready set is bigger than the window that came back (the bumped + // window, or the server's default page for an unlimited fetch), so + // the window cannot say how many of its unseen rows are blocked. Nor + // can refsByIssue: it names every workspace issue with an unsatisfied + // external blocker, closed ones and ones the filter excludes included, + // and only those in the filtered ready set were ever in total. Size + // that set instead. bd ready's own page (above) is exact either way. + adjustedTotal, err = s.refetchedReadyTotal(ctx, filter, total, blockedIDs) + if err != nil { + return nil, 0, err + } + } + if adjustedTotal < len(kept) { + adjustedTotal = len(kept) + } + if adjustedTotal < 0 { + adjustedTotal = 0 + } + return kept, adjustedTotal, nil +} + +// refetchedReadyTotal sizes the externally filtered ready set when the first +// window held only part of it: it refetches the whole filtered set, ids only, +// and subtracts the blocked rows actually in it. A refetch that comes back +// truncated too (the set grew in between, or the server bounded the page) +// falls back to subtracting every blocked id, the bound least likely to +// overstate how much ready work remains. +func (s *Store) refetchedReadyTotal(ctx context.Context, filter types.WorkFilter, total int, blockedIDs map[string]bool) (int, error) { + full := filter + full.Offset = 0 + full.Limit = total + len(blockedIDs) + full.MaxRows = 0 + full.MaxRowsSource = "" + full.Lite = true + rows, fullTotal, err := s.inner.GetReadyWorkWithCountsAndTotal(ctx, full) + if err != nil { + return 0, err + } + if fullTotal > len(rows) { + return total - len(blockedIDs), nil + } + _, blocked := dropBlockedIssuesWithCounts(rows, blockedIDs, 0) + return fullTotal - blocked, nil +} + // CountReadyWork reports the externally filtered ready count. func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (int, error) { filter.Limit = 0 @@ -224,15 +446,14 @@ func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (in if err != nil { return 0, err } - filter = withExternalExclusions(filter, state.refsByIssue) if counter, ok := storage.UnwrapStore(s.inner).(storage.ReadyWorkCounter); ok { - return counter.CountReadyWork(ctx, filter) + return counter.CountReadyWork(ctx, withExternalExclusions(filter, state.refsByIssue)) } - issues, err := s.inner.GetReadyWork(ctx, filter) - if err != nil { - return 0, err - } - return len(issues), nil + // s.inner has no indexed counter (httpclient.Store among others): reuse + // the counts-and-total path above, which already copes with a store whose + // transport cannot express ExcludeIDs at all, nor this zero Limit. + _, total, err := s.readyWorkWithCountsAndTotal(ctx, filter, state) + return total, err } // ReadyCounter sizes the ready set through CountReadyWork above, so the total @@ -399,6 +620,8 @@ func matchesParentFilter(issueID string, parentID *string, allDeps map[string][] } // IsBlocked includes explicit unsatisfied external blockers in the close guard. +// A store that serves its roles refuses the reads this makes, so over one the +// per-issue roles in remote_roles.go answer instead. func (s *Store) IsBlocked(ctx context.Context, issueID string) (bool, []string, error) { blocked, blockers, err := s.inner.IsBlocked(ctx, issueID) if err != nil { @@ -429,7 +652,8 @@ func (s *Store) IsBlocked(ctx context.Context, issueID string) (bool, []string, // IsBlockedBatch preserves the external blocker invariant for batch callers. // The embedded Dolt implementation promotes this method from the wrapped // store, so it must be declared explicitly here rather than relying on -// IsBlocked alone. +// IsBlocked alone. Like IsBlocked, it refuses over a store that serves its +// roles. func (s *Store) IsBlockedBatch(ctx context.Context, issueIDs []string) (map[string]bool, error) { blocked, err := s.inner.IsBlockedBatch(ctx, issueIDs) if err != nil { @@ -463,7 +687,9 @@ func (s *Store) IsBlockedBatch(ctx context.Context, issueIDs []string) (map[stri // CloseIssueChecked applies the external guard before the atomic local close. // The local store cannot see foreign capability state, so promoting its method -// would allow an externally blocked issue to close without --force. +// would allow an externally blocked issue to close without --force. A store +// that serves its roles refuses this method; bd close goes through +// remoteBatchCloser (remote_roles.go) there. func (s *Store) CloseIssueChecked(ctx context.Context, issueID, actor string, opts storage.CloseIssueOptions) (storage.CloseIssueResult, error) { if !opts.Force { issue, err := s.inner.GetIssue(ctx, issueID) @@ -484,7 +710,9 @@ func (s *Store) CloseIssueChecked(ctx context.Context, issueID, actor string, op } // GetDependencyTree appends external refs as synthetic leaf nodes because no -// local issue row exists for the normal graph hydrator to return. +// local issue row exists for the normal graph hydrator to return. A store that +// serves its roles refuses the reads this makes; bd dep tree goes through +// remoteTreeWalker (remote_roles.go) there. func (s *Store) GetDependencyTree(ctx context.Context, issueID string, maxDepth int, showAllPaths bool, reverse bool) ([]*types.TreeNode, error) { tree, err := s.inner.GetDependencyTree(ctx, issueID, maxDepth, showAllPaths, reverse) if err != nil || reverse || len(tree) == 0 { diff --git a/internal/storage/externaldeps/decorator_test.go b/internal/storage/externaldeps/decorator_test.go index 475c379d6..7cd6f3438 100644 --- a/internal/storage/externaldeps/decorator_test.go +++ b/internal/storage/externaldeps/decorator_test.go @@ -3,6 +3,7 @@ package externaldeps import ( "context" "errors" + "fmt" "slices" "testing" "time" @@ -15,22 +16,48 @@ import ( type fakeStore struct { storage.DoltStorage - ready []*types.Issue - blocked []*types.BlockedIssue - tree []*types.TreeNode - deps map[string][]*types.Dependency - labels map[string][]*types.Issue - labelErr error - claimed []string - isBlocked bool - blockerIDs []string - closed []string - lifecycle publicops.Lifecycle - batch *fakeBatchCloser + ready []*types.Issue + blocked []*types.BlockedIssue + tree []*types.TreeNode + deps map[string][]*types.Dependency + labels map[string][]*types.Issue + labelErr error + claimed []string + isBlocked bool + blockerIDs []string + closed []string + lifecycle publicops.Lifecycle + batch *fakeBatchCloser + enforced bool + enforcedErr error + + excludeIDsUnsupported bool + // defaultPage, when positive, is how many rows a zero-Limit ready listing + // returns, while GetReadyWorkWithCountsAndTotal's total still counts the + // whole set: httpclient.Store's shape, whose wire omits a zero limit so + // the server answers with its default page. Zero keeps the SQL stores' + // meaning, every row. + defaultPage int } func (f *fakeStore) IssueLifecycle() (publicops.Lifecycle, error) { return f.lifecycle, nil } +// ServerEnforcesExternalDependencyPolicy makes every fakeStore satisfy +// storage.ExternalDependencyPolicyProber. Defaulting to (false, nil) keeps +// every existing test's raw store on today's client-side-enforcement path +// unchanged; enforced:true is set explicitly only by the test below that +// pins loadBlockingState's skip branch. +func (f *fakeStore) ServerEnforcesExternalDependencyPolicy(_ context.Context) (bool, error) { + return f.enforced, f.enforcedErr +} + +// ExcludeIDsUnsupported likewise makes every fakeStore satisfy +// storage.ExcludeIDsUnsupportedStore. Defaulting to false keeps every existing +// test's exclusions folded into WorkFilter.ExcludeIDs; excludeIDsUnsupported: +// true models httpclient.Store, whose ready listings refuse ExcludeIDs (see +// GetReadyWork below), so the decorator must drop blocked rows client-side. +func (f *fakeStore) ExcludeIDsUnsupported() bool { return f.excludeIDsUnsupported } + // BatchCloser without BatchCloserWithPolicy models a backend that predates // storage.PolicyBatchCloserSource. func (f *fakeStore) BatchCloser() (publicops.BatchCloser, error) { return f.batch, nil } @@ -59,13 +86,29 @@ func (f *fakeLifecycle) Update(context.Context, publicops.UpdateRequest) (public } func (f *fakeStore) GetReadyWork(_ context.Context, filter types.WorkFilter) ([]*types.Issue, error) { + candidates, err := f.readyCandidates(filter) + if err != nil { + return nil, err + } + limit := filter.Limit + if limit <= 0 { + limit = f.defaultPage + } + return page(candidates, filter.Offset, limit), nil +} + +// readyCandidates is the whole ready set filter admits, before paging. +func (f *fakeStore) readyCandidates(filter types.WorkFilter) ([]*types.Issue, error) { + if f.excludeIDsUnsupported && len(filter.ExcludeIDs) > 0 { + return nil, errors.New("fakeStore: ExcludeIDs cannot be expressed") + } candidates := make([]*types.Issue, 0, len(f.ready)) for _, issue := range f.ready { if !slices.Contains(filter.ExcludeIDs, issue.ID) { candidates = append(candidates, issue) } } - return page(candidates, filter.Offset, filter.Limit), nil + return candidates, nil } func (f *fakeStore) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, error) { @@ -85,9 +128,7 @@ func (f *fakeStore) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter t if err != nil { return nil, 0, err } - unbounded := filter - unbounded.Limit, unbounded.Offset = 0, 0 - all, err := f.GetReadyWork(ctx, unbounded) + all, err := f.readyCandidates(filter) if err != nil { return nil, 0, err } @@ -242,7 +283,7 @@ func externalDep(source, ref string, depType types.DependencyType) *types.Depend return &types.Dependency{IssueID: source, DependsOnID: ref, Type: depType} } -func testStore(raw, foreign *fakeStore, configured bool) *Store { +func testStore(raw storage.DoltStorage, foreign *fakeStore, configured bool) *Store { return New( raw, func(project ProjectName) (string, bool) { @@ -321,6 +362,46 @@ func TestGetReadyWorkFailsClosedForUnconfiguredProject(t *testing.T) { } } +// TestLoadBlockingStateSkipsClientSideEnforcementWhenServerReportsItEnforced +// pins design 3.6's skip branch in loadBlockingState (internal/storage/ +// externaldeps/decorator.go) on its CURRENT shape: a remote store +// implementing storage.ExternalDependencyPolicyProber that reports +// enforced=true must stop this decorator from applying its own client-side +// exclusion at all. The S6 review's M5 mutation targeted an earlier, cruder +// "skip whenever the backend is remote" implementation that the HIGH-1 fix +// replaced with this probe-based design, so M5's exact mutation target no +// longer exists; this test gives the probe's own skip branch equivalent +// mutation coverage. Nothing else in the suite exercises enforced==true: see +// internal/httpclient/served_external_dependency_policy_test.go, which +// documents exercising only enforced==false against a real served server +// (OSS httpapi never advertises the capability). +// +// configured=false on the foreign-project locator reproduces the fail-closed +// path from TestGetReadyWorkFailsClosedForUnconfiguredProject: if the +// `if enforced { return blockingState{}, nil }` skip were removed or +// inverted, this would fall through to ordinary client-side enforcement, +// which cannot resolve "remote" and would exclude be-a. Seeing be-a survive +// is therefore proof the skip branch fired. +func TestLoadBlockingStateSkipsClientSideEnforcementWhenServerReportsItEnforced(t *testing.T) { + a := issue("be-a") + raw := &fakeStore{ + ready: []*types.Issue{a}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + enforced: true, + } + store := testStore(raw, &fakeStore{}, false) + + got, err := store.GetReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if ids := issueIDs(got); !slices.Equal(ids, []string{a.ID}) { + t.Fatalf("ready IDs = %v, want [%s] (a server-enforced probe should skip this decorator's own client-side exclusion)", ids, a.ID) + } +} + func TestGetReadyWorkFailsClosedWhenForeignReadFails(t *testing.T) { a := issue("be-a") raw := &fakeStore{ready: []*types.Issue{a}, deps: map[string][]*types.Dependency{ @@ -464,6 +545,183 @@ func TestReadyTotalsAgreeAcrossOutputModes(t *testing.T) { } } +// TestReadyTotalsWhenStoreCannotExpressExcludeIDs pins the path httpclient.Store +// takes: the decorator bumps Limit and drops externally blocked rows +// client-side, and the page, its total, and CountReadyWork must all still +// exclude them. be-a comes back first, ahead of the rows kept — the order in +// which compacting the fetched rows in place once overwrote be-a before the +// total's blocked-row count reached it, so the total came out 3, not 2. +func TestReadyTotalsWhenStoreCannotExpressExcludeIDs(t *testing.T) { + a, b, c := issue("be-a"), issue("be-b"), issue("be-c") + raw := &fakeStore{ + ready: []*types.Issue{a, b, c}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + excludeIDsUnsupported: true, + } + store := testStore(raw, &fakeStore{}, true) + + for _, tc := range []struct { + name string + limit int + wantIDs []string + }{ + {name: "unlimited", limit: 0, wantIDs: []string{b.ID, c.ID}}, + // The bumped window (2+1) holds the whole ready set: the total is exact. + {name: "window holds the set", limit: 2, wantIDs: []string{b.ID, c.ID}}, + // The ready set outgrows the bumped window (1+1): the total is the + // inner total less every externally blocked issue. + {name: "set outgrows the window", limit: 1, wantIDs: []string{b.ID}}, + } { + t.Run(tc.name, func(t *testing.T) { + filter := types.WorkFilter{Limit: tc.limit} + issues, err := store.GetReadyWork(t.Context(), filter) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if ids := issueIDs(issues); !slices.Equal(ids, tc.wantIDs) { + t.Fatalf("GetReadyWork IDs = %v, want %v", ids, tc.wantIDs) + } + rows, total, err := store.GetReadyWorkWithCountsAndTotal(t.Context(), filter) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal: %v", err) + } + ids := make([]string, 0, len(rows)) + for _, row := range rows { + ids = append(ids, row.ID) + } + if !slices.Equal(ids, tc.wantIDs) || total != 2 { + t.Fatalf("page = %v, total = %d; want %v, total 2 (be-a is externally blocked)", ids, total, tc.wantIDs) + } + }) + } + + got, err := store.CountReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("CountReadyWork: %v", err) + } + if got != 2 { + t.Fatalf("ready count = %d, want 2 (be-a is externally blocked)", got) + } +} + +// TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage pins the shape +// httpclient.Store has and an every-row Limit==0 fake hid: the wire omits a +// zero Limit, the server answers with its default page, and the total still +// counts the whole ready set. CountReadyWork, text-mode bd ready's total, +// always fetches with Limit 0, so be-z (ready, externally blocked, and past +// that page) once stayed in its total because the decorator took a zero Limit +// for a window that held every row. +func TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage(t *testing.T) { + const defaultPage = 100 // workapi.DefaultReadyLimit, the server's page for an omitted limit + ready := make([]*types.Issue, 0, defaultPage+2) + for i := range defaultPage + 1 { + ready = append(ready, issue(fmt.Sprintf("be-%03d", i))) + } + z := issue("be-z") + raw := &fakeStore{ + ready: append(ready, z), + deps: map[string][]*types.Dependency{ + z.ID: {externalDep(z.ID, "external:remote:payments", types.DepBlocks)}, + }, + excludeIDsUnsupported: true, + defaultPage: defaultPage, + } + store := testStore(raw, &fakeStore{}, true) + const want = defaultPage + 1 // every ready issue but be-z + + counter, err := store.ReadyCounter() + if err != nil { + t.Fatalf("ReadyCounter: %v", err) + } + text, err := counter.CountReady(t.Context(), publicops.ReadyRequest{}) + if err != nil { + t.Fatalf("CountReady: %v", err) + } + if text.Total != want { + t.Fatalf("text total = %d, want %d (be-z is externally blocked)", text.Total, want) + } + // The JSON total must agree, whether the page is unlimited (the server's + // default page comes back) or the CLI's default limit (bumped by one). + for _, limit := range []int{0, defaultPage} { + rows, total, err := store.GetReadyWorkWithCountsAndTotal(t.Context(), types.WorkFilter{Limit: limit}) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal(Limit %d): %v", limit, err) + } + if len(rows) != defaultPage || total != want { + t.Fatalf("Limit %d: %d rows, total %d; want %d rows, total %d (be-z is externally blocked)", limit, len(rows), total, defaultPage, want) + } + } +} + +// TestReadyTotalsIgnoreExternallyBlockedIssuesOutsideTheReadySet pins the +// total of a ready set bigger than the window that came back. refsByIssue +// names every workspace issue with an unsatisfied external blocker, closed +// ones and ones the filter excludes included, but only those in the filtered +// ready set were ever in the inner total, so subtracting all of them once +// reported 100 of 101 ready issues as soon as an issue outside the ready set +// held an external blocker. +func TestReadyTotalsIgnoreExternallyBlockedIssuesOutsideTheReadySet(t *testing.T) { + const defaultPage = 100 // workapi.DefaultReadyLimit, the server's page for an omitted limit + for _, tc := range []struct { + name string + // readyBlocked adds be-z: ready, externally blocked, and past the page. + readyBlocked bool + // outside is how many externally blocked issues are not ready. + outside int + }{ + {name: "one blocked issue outside the ready set", outside: 1}, + {name: "blocked issues inside and outside the ready set", readyBlocked: true, outside: 20}, + } { + t.Run(tc.name, func(t *testing.T) { + ready := make([]*types.Issue, 0, defaultPage+2) + for i := range defaultPage + 1 { + ready = append(ready, issue(fmt.Sprintf("be-%03d", i))) + } + deps := make(map[string][]*types.Dependency) + if tc.readyBlocked { + z := issue("be-z") + ready = append(ready, z) + deps[z.ID] = []*types.Dependency{externalDep(z.ID, "external:remote:payments", types.DepBlocks)} + } + for i := range tc.outside { + id := fmt.Sprintf("be-done-%02d", i) + deps[id] = []*types.Dependency{externalDep(id, "external:remote:payments", types.DepBlocks)} + } + raw := &fakeStore{ + ready: ready, + deps: deps, + excludeIDsUnsupported: true, + defaultPage: defaultPage, + } + store := testStore(raw, &fakeStore{}, true) + const want = defaultPage + 1 // every ready issue but be-z + + counter, err := store.ReadyCounter() + if err != nil { + t.Fatalf("ReadyCounter: %v", err) + } + text, err := counter.CountReady(t.Context(), publicops.ReadyRequest{}) + if err != nil { + t.Fatalf("CountReady: %v", err) + } + if text.Total != want { + t.Fatalf("text total = %d, want %d", text.Total, want) + } + for _, limit := range []int{0, defaultPage} { + rows, total, err := store.GetReadyWorkWithCountsAndTotal(t.Context(), types.WorkFilter{Limit: limit}) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal(Limit %d): %v", limit, err) + } + if len(rows) != defaultPage || total != want { + t.Fatalf("Limit %d: %d rows, total %d; want %d rows, total %d", limit, len(rows), total, defaultPage, want) + } + } + }) + } +} + func TestGetBlockedIssuesAddsUnsatisfiedExternalRefs(t *testing.T) { a, c := issue("be-a"), issue("be-c") raw := &fakeStore{ diff --git a/internal/storage/externaldeps/remote_roles.go b/internal/storage/externaldeps/remote_roles.go new file mode 100644 index 000000000..bc89f791d --- /dev/null +++ b/internal/storage/externaldeps/remote_roles.go @@ -0,0 +1,412 @@ +package externaldeps + +import ( + "context" + "errors" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// Remote role composition. +// +// The base accessors (role_accessors.go, batch_closer.go) rebuild each role +// over this store's legacy methods, which a SQL store implements in full. A +// remote store does not: httpclient.Store serves its roles over the wire and +// refuses most of the legacy methods those builds call (SearchIssuesWithCounts, +// IsBlocked, GetDependencyRecordsForIssues, GetDependencyTree, +// ClaimReadyIssue, BatchCloserWithPolicy), so the base accessors would fail +// `bd list`, `bd dep tree`, `bd ready --claim` and a policed close even in a +// workspace with no external dependency at all. Over such a store each +// accessor starts from the inner store's own role instead and layers the +// policy around it, before or after the served call. +// +// storage.RemoteBackendStore chooses that composition and nothing else. +// Whether the policy runs is still serverEnforcesPolicy's question (design +// 3.6), asked on every path that loads blocking state. + +// servesRoles reports whether the inner store serves its roles rather than +// building them over its legacy methods. +func (s *Store) servesRoles() bool { + remote, ok := storage.UnwrapStore(s.inner).(storage.RemoteBackendStore) + return ok && remote.IsRemoteBackendStore() +} + +// servedEdgeAnchors is the most anchors one served edge read names: the wire +// caps listDependencies' issue_id at 100 items, and the client passes a +// request through without chunking it. +const servedEdgeAnchors = 100 + +// servedEdges reads the stored outgoing edges of ids through the inner store's +// EdgeReader, the served stand-in for GetDependencyRecordsForIssues. A missing +// anchor contributes no edges. +func (s *Store) servedEdges(ctx context.Context, ids []string) (map[string][]*types.Dependency, error) { + anchors := make([]string, 0, len(ids)) + named := make(map[string]bool, len(ids)) + for _, id := range ids { + if id != "" && !named[id] { + named[id] = true + anchors = append(anchors, id) + } + } + deps := make(map[string][]*types.Dependency, len(anchors)) + if len(anchors) == 0 { + return deps, nil + } + reader, err := s.inner.EdgeReader() + if err != nil { + return nil, err + } + for start := 0; start < len(anchors); start += servedEdgeAnchors { + end := min(start+servedEdgeAnchors, len(anchors)) + result, err := reader.ReadEdges(ctx, issueops.EdgeReadRequest{IDs: anchors[start:end]}) + if err != nil { + return nil, err + } + for _, anchor := range result.Anchors { + deps[anchor.ID] = append(deps[anchor.ID], anchor.Edges...) + } + } + return deps, nil +} + +// blockingStateFor is loadBlockingState narrowed to ids, for a role that names +// its issues and so needs only their edges. +func (s *Store) blockingStateFor(ctx context.Context, ids []string) (blockingState, error) { + if enforced, err := s.serverEnforcesPolicy(ctx); err != nil || enforced { + return blockingState{}, err + } + deps, err := s.servedEdges(ctx, ids) + if err != nil { + return blockingState{}, fmt.Errorf("external dependencies: list blocking records: %w", err) + } + return s.blockingStateFromRecords(ctx, deps) +} + +// plainDownTree reports whether req is the walk the policy decorates. Reverse +// walks do not follow a source's dependencies, and the status- or row-bounded +// variants keep the backend's own traversal semantics. +func plainDownTree(req issueops.WalkTreeRequest) bool { + return (req.Direction == "" || req.Direction == issueops.TreeDown) && req.Status == "" && req.MaxRows == 0 +} + +// isLostClaimRace classifies the refusals that mean someone else took a +// candidate first. ErrNotFound joins them, as in httpclient's own composed +// claim: a ready row deleted before it was claimed is the same situation for +// a caller that wants the next piece of work. +func isLostClaimRace(err error) bool { + return errors.Is(err, issueops.ErrAlreadyClaimed) || + errors.Is(err, issueops.ErrNotClaimable) || + errors.Is(err, issueops.ErrNotFound) +} + +// remoteReader serves List and Get from the inner store's reader: this store +// overrides no search or detail method, so neither loses any policy. Ready and +// a ready-flagged List go through the policy reader, whose ready listing drops +// externally blocked work. +type remoteReader struct { + served issueops.Reader + policy issueops.Reader +} + +func (r *remoteReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issueops.IssuePage, error) { + return r.policy.Ready(ctx, req) +} + +func (r *remoteReader) List(ctx context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + if req.ReadyFlag { + return r.policy.List(ctx, req) + } + return r.served.List(ctx, req) +} + +func (r *remoteReader) Get(ctx context.Context, req issueops.GetRequest) (*issueops.IssueDetails, error) { + return r.served.Get(ctx, req) +} + +// remoteClaimer refuses an externally blocked claim before the served one. +// Local blockers are the served claim's own answer: IsBlocked is a refused +// legacy method on a remote store, and the server checks its own graph. +type remoteClaimer struct { + served issueops.Claimer + policy *Store +} + +func (c *remoteClaimer) Claim(ctx context.Context, req issueops.ClaimRequest) (issueops.ClaimResult, error) { + state, err := c.policy.blockingStateFor(ctx, []string{req.IssueID}) + if err != nil { + return issueops.ClaimResult{}, err + } + if refs := state.refsByIssue[req.IssueID]; len(refs) > 0 { + return issueops.ClaimResult{}, fmt.Errorf("%w: %s is blocked by %v", storage.ErrCloseBlocked, req.IssueID, refs) + } + return c.served.Claim(ctx, req) +} + +// remoteClaimPage is how many externally unblocked candidates one pass of +// remoteReadyClaimer reads, matching httpclient's composed ready claim. +const remoteClaimPage = 25 + +// remoteReadyClaimer hands the claim to the served ReadyClaimer when no issue +// in the workspace holds an unsatisfied external ref. Otherwise the served +// claim cannot be told what to skip — the wire's claim-next has no exclusion +// parameter — so it reads the policy's filtered ready page and claims down it +// through the served Claimer, as httpclient's own composition does for an +// older server. The same window applies: a row that gains a blocker between +// the listing and its claim can still be claimed. +// +// A closed holder counts too, as it does in every backend's blocking state. +// It can never be a candidate, so the composed claim takes the same issue the +// served one would. Leaving it out would cost a status read per holder, since +// the wire has no batch form of one. +type remoteReadyClaimer struct{ policy *Store } + +func (c *remoteReadyClaimer) ClaimNext(ctx context.Context, req issueops.ClaimNextRequest) (issueops.ClaimNextResult, error) { + if err := storageissueops.ValidateClaimNextRequest(req); err != nil { + return issueops.ClaimNextResult{}, err + } + filter, err := workapi.BuildReadyFilter(req.Filter) + if err != nil { + return issueops.ClaimNextResult{}, err + } + state, err := c.policy.loadBlockingState(ctx) + if err != nil { + return issueops.ClaimNextResult{}, err + } + if len(state.refsByIssue) == 0 { + served, err := c.policy.inner.ReadyClaimer() + if err != nil { + return issueops.ClaimNextResult{}, err + } + return served.ClaimNext(ctx, req) + } + claimer, err := c.policy.inner.IssueClaimer() + if err != nil { + return issueops.ClaimNextResult{}, err + } + // The candidates a local claim-next selects (ClaimReadyIssueInTx): open, + // unassigned, with no row cap on a scan that delivers one row. + filter.Status = types.StatusOpen + filter.Unassigned = true + filter.Assignee = nil + filter.MaxRows = 0 + filter.MaxRowsSource = "" + filter.Limit = remoteClaimPage + // A second pass only after a full page was lost: claimed rows leave the + // unassigned set, so it reads the candidates behind them. + var lost int + for range 2 { + rows, err := c.policy.readyWorkWithCounts(ctx, filter, state) + if err != nil { + return issueops.ClaimNextResult{}, err + } + if len(rows) == 0 { + return issueops.ClaimNextResult{}, nil + } + for _, row := range rows { + if row == nil || row.Issue == nil { + continue + } + res, err := claimer.Claim(ctx, issueops.ClaimRequest{Actor: req.Actor, IssueID: row.ID}) + switch { + case err == nil: + return issueops.ClaimNextResult{Claimed: &types.IssueWithCounts{ + Issue: res.Issue, + DependencyCount: row.DependencyCount, + DependentCount: row.DependentCount, + CommentCount: row.CommentCount, + Parent: row.Parent, + }}, nil + case isLostClaimRace(err): + lost++ + default: + return issueops.ClaimNextResult{}, err + } + } + if len(rows) < remoteClaimPage { + break + } + } + return issueops.ClaimNextResult{}, fmt.Errorf("claim ready: lost %d races for externally unblocked ready work; re-run to take the next one", lost) +} + +// remoteBlockingAnnotator merges the external blockers the policy resolves +// into the served annotation's BlockedBy. +type remoteBlockingAnnotator struct { + served issueops.BlockingAnnotator + policy *Store +} + +func (a *remoteBlockingAnnotator) AnnotateBlocking(ctx context.Context, req issueops.BlockingRequest) (issueops.BlockingResult, error) { + result, err := a.served.AnnotateBlocking(ctx, req) + if err != nil || len(result.Items) == 0 { + return result, err + } + ids := make([]string, 0, len(result.Items)) + for _, item := range result.Items { + ids = append(ids, item.ID) + } + state, err := a.policy.blockingStateFor(ctx, ids) + if err != nil { + return issueops.BlockingResult{}, err + } + for i := range result.Items { + for _, ref := range state.refsByIssue[result.Items[i].ID] { + result.Items[i].BlockedBy = appendUnique(result.Items[i].BlockedBy, ref) + } + } + return result, nil +} + +// remoteTreeWalker adds external leaves to the served down-tree. A leaf the +// server already rendered is kept as it came (appendTreeExternalReferences +// skips an id the tree holds), and like GetDependencyTree the leaves are +// rendered whether or not the server enforces the policy: a leaf is display, +// not enforcement. +type remoteTreeWalker struct { + served issueops.TreeWalker + policy *Store +} + +func (t *remoteTreeWalker) WalkTree(ctx context.Context, req issueops.WalkTreeRequest) (issueops.TreeResult, error) { + result, err := t.served.WalkTree(ctx, req) + if err != nil || !plainDownTree(req) || len(result.Nodes) == 0 { + return result, err + } + ids := make([]string, 0, len(result.Nodes)) + for _, node := range result.Nodes { + if node != nil && !isExternalReference(node.ID) { + ids = append(ids, node.ID) + } + } + deps, err := t.policy.servedEdges(ctx, ids) + if err != nil { + return issueops.TreeResult{}, fmt.Errorf("external dependencies: load tree edges: %w", err) + } + nodes, err := t.policy.appendTreeExternalReferences(ctx, result.Nodes, deps, req.MaxDepth, false) + if err != nil { + return issueops.TreeResult{}, err + } + result.Nodes = nodes + return result, nil +} + +// remoteBatchCloser refuses each externally blocked, still-open item the way +// the local policy close does, then sends the rest to the served closer as one +// request, so they still close in one server transaction. The external answer +// is read before that request rather than inside it, the same window +// remoteReadyClaimer has. +// +// A batch that also claims the next issue is refused whole, before anything +// closes. The local closer claims inside the closing transaction with the +// policy's exclusions in its filter. A served close has no parameter to carry +// them, and claiming after the close commits would split the role's one +// transaction in two. The refusal does not read the workspace, so the command +// fails the same way whether or not anything in it is externally blocked. +type remoteBatchCloser struct{ policy *Store } + +func (c *remoteBatchCloser) CloseBatch(ctx context.Context, request issueops.CloseBatchRequest) (issueops.CloseBatchResult, error) { + if request.ClaimNext != nil { + return c.closeAndClaimNext(ctx, request) + } + if request.Force { + // Force waives the policy, so the base closer reaches the served one + // with an empty policy. + return (&batchCloser{policy: c.policy}).CloseBatch(ctx, request) + } + if err := storageissueops.ValidateCloseBatchRequest(request); err != nil { + return issueops.CloseBatchResult{}, err + } + ids := make([]string, 0, len(request.Items)) + for _, item := range request.Items { + ids = append(ids, item.IssueID) + } + state, err := c.policy.blockingStateFor(ctx, ids) + if err != nil { + return issueops.CloseBatchResult{}, err + } + served, err := c.policy.inner.BatchCloser() + if err != nil { + return issueops.CloseBatchResult{}, err + } + if len(state.refsByIssue) == 0 { + return served.CloseBatch(ctx, request) + } + + policy := storage.NewBatchClosePolicy(state.refsByIssue) + outcomes := make([]issueops.CloseOutcome, len(request.Items)) + forward := request + forward.Items = make([]issueops.BatchCloseItem, 0, len(request.Items)) + forwarded := make([]int, 0, len(request.Items)) + for i, item := range request.Items { + if refusal := policy.CheckClose(item.IssueID, false); refusal != nil { + // As in the local close: only a live target is refused, so an + // idempotent re-close and a not-found keep their own answers. + issue, err := c.policy.inner.GetIssue(ctx, item.IssueID) + if err != nil { + outcomes[i] = issueops.CloseOutcome{IssueID: item.IssueID, Err: err} + continue + } + if issue != nil && issue.Status != types.StatusClosed { + outcomes[i] = issueops.CloseOutcome{IssueID: item.IssueID, Err: refusal} + continue + } + } + forward.Items = append(forward.Items, item) + forwarded = append(forwarded, i) + } + if len(forward.Items) == 0 { + return issueops.CloseBatchResult{Outcomes: outcomes}, nil + } + result, err := served.CloseBatch(ctx, forward) + if err != nil { + return issueops.CloseBatchResult{}, err + } + if len(result.Outcomes) != len(forward.Items) { + return issueops.CloseBatchResult{}, fmt.Errorf("close batch: served closer answered %d outcomes for %d items", len(result.Outcomes), len(forward.Items)) + } + for j, outcome := range result.Outcomes { + outcomes[forwarded[j]] = outcome + } + return issueops.CloseBatchResult{Outcomes: outcomes}, nil +} + +// closeAndClaimNext answers a batch with a next claim. A request that is +// invalid on every backend is still ErrValidation. A server that enforces the +// policy itself gets the request as it came; over the v0 wire its closer +// refuses a next claim too (ledger row W-CloseBatchRequest.ClaimNext). +func (c *remoteBatchCloser) closeAndClaimNext(ctx context.Context, request issueops.CloseBatchRequest) (issueops.CloseBatchResult, error) { + if err := storageissueops.ValidateCloseBatchRequest(request); err != nil { + return issueops.CloseBatchResult{}, err + } + if _, err := workapi.BuildReadyFilter(*request.ClaimNext); err != nil { + return issueops.CloseBatchResult{}, err + } + enforced, err := c.policy.serverEnforcesPolicy(ctx) + if err != nil { + return issueops.CloseBatchResult{}, err + } + if !enforced { + return issueops.CloseBatchResult{}, fmt.Errorf( + "close with a next claim: the served batch close cannot carry a next claim, so nothing was closed; close without --claim-next, then run `bd ready --claim` (%w)", + &storage.ErrUnsupported{Op: "CloseBatchRequest.ClaimNext", Backend: fmt.Sprintf("%T", storage.UnwrapStore(c.policy.inner))}) + } + served, err := c.policy.inner.BatchCloser() + if err != nil { + return issueops.CloseBatchResult{}, err + } + return served.CloseBatch(ctx, request) +} + +var ( + _ issueops.Reader = (*remoteReader)(nil) + _ issueops.Claimer = (*remoteClaimer)(nil) + _ issueops.ReadyClaimer = (*remoteReadyClaimer)(nil) + _ issueops.BlockingAnnotator = (*remoteBlockingAnnotator)(nil) + _ issueops.TreeWalker = (*remoteTreeWalker)(nil) + _ issueops.BatchCloser = (*remoteBatchCloser)(nil) +) diff --git a/internal/storage/externaldeps/remote_roles_test.go b/internal/storage/externaldeps/remote_roles_test.go new file mode 100644 index 000000000..793d9bb58 --- /dev/null +++ b/internal/storage/externaldeps/remote_roles_test.go @@ -0,0 +1,636 @@ +package externaldeps + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// remoteStore models httpclient.Store for remote_roles.go. Each role it hands +// out is served over the embedded fakeStore's rows, the way a server that +// knows nothing of the policy answers it, and each legacy method a base +// accessor builds on is refused, so a composition that reaches one fails the +// test instead of passing on a fake that answers it. Like httpclient.Store it +// cannot express ExcludeIDs and has no BatchCloserWithPolicy. +type remoteStore struct { + *fakeStore + // served logs each served role call, in order. + served []string + // closes logs the item ids of each served batch close. + closes [][]string + // edgeReads logs how many anchors each served edge read named. + edgeReads []int + // scans counts the whole-workspace dependency scans read through it. + scans int + // localBlockers is the served annotation's own BlockedBy, per id. + localBlockers map[string][]string + // claimErr, when set, answers every served Claim. + claimErr error +} + +var errLegacyRefused = errors.New("remoteStore: legacy method refused") + +// wireEdgeAnchorCap is the most anchors the wire's listDependencies accepts. +const wireEdgeAnchorCap = 100 + +func newRemoteStore(ready ...*types.Issue) *remoteStore { + return &remoteStore{fakeStore: &fakeStore{ + ready: ready, + deps: make(map[string][]*types.Dependency), + excludeIDsUnsupported: true, + }} +} + +func (r *remoteStore) IsRemoteBackendStore() bool { return true } + +// httpclient.Store serves the vocabulary a listing loads before its search. +func (r *remoteStore) GetCustomStatusesDetailed(context.Context) ([]types.CustomStatus, error) { + return nil, nil +} +func (r *remoteStore) GetCustomTypes(context.Context) ([]string, error) { return nil, nil } +func (r *remoteStore) GetInfraTypes(context.Context) map[string]bool { return nil } + +func (r *remoteStore) SearchIssuesWithCounts(context.Context, string, types.IssueFilter) ([]*types.IssueWithCounts, error) { + return nil, errLegacyRefused +} + +func (r *remoteStore) IsBlocked(context.Context, string) (bool, []string, error) { + return false, nil, errLegacyRefused +} + +func (r *remoteStore) GetDependencyRecordsForIssues(context.Context, []string) (map[string][]*types.Dependency, error) { + return nil, errLegacyRefused +} + +func (r *remoteStore) GetDependencyTree(context.Context, string, int, bool, bool) ([]*types.TreeNode, error) { + return nil, errLegacyRefused +} + +func (r *remoteStore) ClaimReadyIssue(context.Context, types.WorkFilter, string) (*types.Issue, error) { + return nil, errLegacyRefused +} + +func (r *remoteStore) GetAllDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + r.scans++ + return r.fakeStore.GetAllDependencyRecords(ctx) +} + +func (r *remoteStore) GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + r.scans++ + return r.fakeStore.GetExternalBlockingDependencyRecords(ctx) +} + +func (r *remoteStore) IssueReader() (publicops.Reader, error) { return servedReader{r}, nil } +func (r *remoteStore) IssueClaimer() (publicops.Claimer, error) { return servedClaimer{r}, nil } +func (r *remoteStore) ReadyClaimer() (publicops.ReadyClaimer, error) { + return servedReadyClaimer{r}, nil +} +func (r *remoteStore) TreeWalker() (publicops.TreeWalker, error) { return servedWalker{r}, nil } +func (r *remoteStore) EdgeReader() (publicops.EdgeReader, error) { return servedEdgeReader{r}, nil } +func (r *remoteStore) BatchCloser() (publicops.BatchCloser, error) { return servedCloser{r}, nil } + +func (r *remoteStore) BlockingAnnotator() (publicops.BlockingAnnotator, error) { + return servedAnnotator{r}, nil +} + +type servedReader struct{ s *remoteStore } + +func (r servedReader) Ready(context.Context, publicops.ReadyRequest) (publicops.IssuePage, error) { + return publicops.IssuePage{}, errors.New("served Ready: ready work is the policy reader's to answer") +} + +func (r servedReader) List(context.Context, publicops.ListRequest) (publicops.IssuePage, error) { + r.s.served = append(r.s.served, "List") + items := make([]*types.IssueWithCounts, 0, len(r.s.ready)) + for _, issue := range r.s.ready { + items = append(items, &types.IssueWithCounts{Issue: issue}) + } + return publicops.IssuePage{Items: items}, nil +} + +func (r servedReader) Get(ctx context.Context, req publicops.GetRequest) (*publicops.IssueDetails, error) { + r.s.served = append(r.s.served, "Get "+req.ID) + issue, err := r.s.GetIssue(ctx, req.ID) + if err != nil { + return nil, err + } + if issue == nil { + return nil, publicops.ErrNotFound + } + return &publicops.IssueDetails{Issue: *issue}, nil +} + +type servedClaimer struct{ s *remoteStore } + +func (c servedClaimer) Claim(ctx context.Context, req publicops.ClaimRequest) (publicops.ClaimResult, error) { + c.s.served = append(c.s.served, "Claim "+req.IssueID) + if c.s.claimErr != nil { + return publicops.ClaimResult{}, c.s.claimErr + } + if err := c.s.ClaimIssue(ctx, req.IssueID, req.Actor); err != nil { + return publicops.ClaimResult{}, err + } + issue, err := c.s.GetIssue(ctx, req.IssueID) + return publicops.ClaimResult{Issue: issue, Changed: true}, err +} + +// servedReadyClaimer claims the first unassigned ready row, blind to the +// policy as a claim-next with no exclusion parameter is. +type servedReadyClaimer struct{ s *remoteStore } + +func (c servedReadyClaimer) ClaimNext(ctx context.Context, req publicops.ClaimNextRequest) (publicops.ClaimNextResult, error) { + c.s.served = append(c.s.served, "ClaimNext") + for _, issue := range c.s.ready { + if issue.Assignee != "" { + continue + } + if err := c.s.ClaimIssue(ctx, issue.ID, req.Actor); err != nil { + return publicops.ClaimNextResult{}, err + } + return publicops.ClaimNextResult{Claimed: &types.IssueWithCounts{Issue: issue}}, nil + } + return publicops.ClaimNextResult{}, nil +} + +type servedAnnotator struct{ s *remoteStore } + +func (a servedAnnotator) AnnotateBlocking(_ context.Context, req publicops.BlockingRequest) (publicops.BlockingResult, error) { + a.s.served = append(a.s.served, "AnnotateBlocking") + items := make([]publicops.IssueBlocking, 0, len(req.IDs)) + for _, id := range req.IDs { + items = append(items, publicops.IssueBlocking{ID: id, BlockedBy: slices.Clone(a.s.localBlockers[id])}) + } + return publicops.BlockingResult{Items: items}, nil +} + +type servedWalker struct{ s *remoteStore } + +func (w servedWalker) WalkTree(context.Context, publicops.WalkTreeRequest) (publicops.TreeResult, error) { + w.s.served = append(w.s.served, "WalkTree") + return publicops.TreeResult{Nodes: slices.Clone(w.s.tree)}, nil +} + +type servedEdgeReader struct{ s *remoteStore } + +func (e servedEdgeReader) ReadEdges(_ context.Context, req publicops.EdgeReadRequest) (publicops.EdgeReadResult, error) { + if len(req.IDs) > wireEdgeAnchorCap { + return publicops.EdgeReadResult{}, fmt.Errorf("served ReadEdges: %d anchors, past the wire's %d", len(req.IDs), wireEdgeAnchorCap) + } + e.s.edgeReads = append(e.s.edgeReads, len(req.IDs)) + anchors := make([]publicops.AnchorEdges, 0, len(req.IDs)) + for _, id := range req.IDs { + anchors = append(anchors, publicops.AnchorEdges{ID: id, Edges: e.s.deps[id]}) + } + return publicops.EdgeReadResult{Anchors: anchors}, nil +} + +type servedCloser struct{ s *remoteStore } + +// errServedClaimNextRefused is the served closer's answer to a next claim. +// httpclient refuses one on every server: the v0 wire's batch close has no +// member to carry it (ledger row W-CloseBatchRequest.ClaimNext). +var errServedClaimNextRefused = &storage.ErrUnsupported{Op: "BatchCloser.CloseBatch", Backend: "http"} + +func (c servedCloser) CloseBatch(ctx context.Context, req publicops.CloseBatchRequest) (publicops.CloseBatchResult, error) { + if req.ClaimNext != nil { + c.s.served = append(c.s.served, "CloseBatch ClaimNext") + return publicops.CloseBatchResult{}, errServedClaimNextRefused + } + ids := make([]string, 0, len(req.Items)) + outcomes := make([]publicops.CloseOutcome, 0, len(req.Items)) + for _, item := range req.Items { + ids = append(ids, item.IssueID) + result, err := c.s.CloseIssueChecked(ctx, item.IssueID, req.Actor, storage.CloseIssueOptions{}) + outcomes = append(outcomes, publicops.CloseOutcome{IssueID: item.IssueID, Changed: err == nil && !result.Unchanged, Err: err}) + } + c.s.closes = append(c.s.closes, ids) + return publicops.CloseBatchResult{Outcomes: outcomes}, nil +} + +const paymentsRef = "external:remote:payments" + +func blockOnPayments(raw *remoteStore, ids ...string) { + for _, id := range ids { + raw.deps[id] = []*types.Dependency{externalDep(id, paymentsRef, types.DepBlocks)} + } +} + +func closeItems(ids ...string) []publicops.BatchCloseItem { + items := make([]publicops.BatchCloseItem, 0, len(ids)) + for _, id := range ids { + items = append(items, publicops.BatchCloseItem{IssueID: id}) + } + return items +} + +// TestRemoteRolesAnswerWithoutTheLegacyMethods pins the blocker the base +// accessors had over httpclient.Store: built over legacy methods a remote +// store refuses, they failed bd list, bd show, bd dep tree, bd ready --claim +// and bd close in a workspace where nothing had an external dependency. +func TestRemoteRolesAnswerWithoutTheLegacyMethods(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := newRemoteStore(a, b) + raw.tree = []*types.TreeNode{{Issue: *a}} + store := testStore(raw, &fakeStore{}, true) + ctx := t.Context() + + reader, err := store.IssueReader() + if err != nil { + t.Fatalf("IssueReader: %v", err) + } + if page, err := reader.List(ctx, publicops.ListRequest{}); err != nil || len(page.Items) != 2 { + t.Fatalf("List = %d items, %v; want 2", len(page.Items), err) + } + if details, err := reader.Get(ctx, publicops.GetRequest{ID: a.ID}); err != nil || details.ID != a.ID { + t.Fatalf("Get = %v, %v; want %s", details, err, a.ID) + } + if page, err := reader.Ready(ctx, publicops.ReadyRequest{}); err != nil || len(page.Items) != 2 { + t.Fatalf("Ready = %d items, %v; want 2", len(page.Items), err) + } + + claimer, err := store.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer: %v", err) + } + if _, err := claimer.Claim(ctx, publicops.ClaimRequest{Actor: "worker", IssueID: a.ID}); err != nil { + t.Fatalf("Claim: %v", err) + } + readyClaimer, err := store.ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer: %v", err) + } + if next, err := readyClaimer.ClaimNext(ctx, publicops.ClaimNextRequest{Actor: "worker"}); err != nil || next.Claimed == nil || next.Claimed.ID != b.ID { + t.Fatalf("ClaimNext = %+v, %v; want %s", next.Claimed, err, b.ID) + } + + annotator, err := store.BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator: %v", err) + } + if result, err := annotator.AnnotateBlocking(ctx, publicops.BlockingRequest{IDs: []string{a.ID, b.ID}}); err != nil || len(result.Items) != 2 { + t.Fatalf("AnnotateBlocking = %+v, %v; want 2 items", result.Items, err) + } + walker, err := store.TreeWalker() + if err != nil { + t.Fatalf("TreeWalker: %v", err) + } + if tree, err := walker.WalkTree(ctx, publicops.WalkTreeRequest{RootID: a.ID, MaxDepth: 50}); err != nil || len(tree.Nodes) != 1 { + t.Fatalf("WalkTree = %d nodes, %v; want the served root alone", len(tree.Nodes), err) + } + + closer, err := store.BatchCloser() + if err != nil { + t.Fatalf("BatchCloser: %v", err) + } + result, err := closer.CloseBatch(ctx, publicops.CloseBatchRequest{Actor: "worker", Items: closeItems(a.ID, b.ID)}) + if err != nil || len(result.Outcomes) != 2 { + t.Fatalf("CloseBatch = %+v, %v; want 2 outcomes", result.Outcomes, err) + } + + if want := []string{"List", "Get be-a", "Claim be-a", "ClaimNext", "AnnotateBlocking", "WalkTree"}; !slices.Equal(raw.served, want) { + t.Fatalf("served calls = %v, want %v", raw.served, want) + } + if want := [][]string{{a.ID, b.ID}}; !slices.EqualFunc(raw.closes, want, slices.Equal[[]string]) { + t.Fatalf("served closes = %v, want %v", raw.closes, want) + } +} + +// TestRemoteReaderPolicesReadyWorkOnly: an externally blocked issue leaves the +// ready listing and stays in a plain one, which claims nothing about +// readiness. +func TestRemoteReaderPolicesReadyWorkOnly(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := newRemoteStore(a, b) + blockOnPayments(raw, a.ID) + reader, err := testStore(raw, &fakeStore{}, true).IssueReader() + if err != nil { + t.Fatalf("IssueReader: %v", err) + } + + ready, err := reader.Ready(t.Context(), publicops.ReadyRequest{}) + if err != nil { + t.Fatalf("Ready: %v", err) + } + if len(ready.Items) != 1 || ready.Items[0].ID != b.ID { + t.Fatalf("Ready = %d items, want %s alone (%s is externally blocked)", len(ready.Items), b.ID, a.ID) + } + listed, err := reader.List(t.Context(), publicops.ListRequest{}) + if err != nil { + t.Fatalf("List: %v", err) + } + if len(listed.Items) != 2 { + t.Fatalf("List = %d items, want both", len(listed.Items)) + } +} + +func TestRemoteClaimerRefusesExternallyBlockedWork(t *testing.T) { + for _, enforced := range []bool{false, true} { + t.Run(fmt.Sprintf("server enforces %v", enforced), func(t *testing.T) { + a := issue("be-a") + raw := newRemoteStore(a) + raw.enforced = enforced + blockOnPayments(raw, a.ID) + claimer, err := testStore(raw, &fakeStore{}, true).IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer: %v", err) + } + + _, err = claimer.Claim(t.Context(), publicops.ClaimRequest{Actor: "worker", IssueID: a.ID}) + if enforced { + // The server already applied the policy, so the claim is its + // to answer. + if err != nil || !slices.Equal(raw.served, []string{"Claim be-a"}) { + t.Fatalf("Claim = %v, served %v; want the served claim", err, raw.served) + } + return + } + if !errors.Is(err, storage.ErrCloseBlocked) || !strings.Contains(err.Error(), paymentsRef) { + t.Fatalf("Claim = %v, want ErrCloseBlocked naming %s", err, paymentsRef) + } + if len(raw.served) != 0 { + t.Fatalf("served calls = %v, want none for a refused claim", raw.served) + } + }) + } +} + +// TestRemoteReadyClaimerSkipsExternallyBlockedWork: the served claim-next +// cannot be told what to skip, so with anything externally blocked the claim +// walks the policy's ready page through the served Claimer instead. be-a sorts +// first, where the served claim-next would take it. +func TestRemoteReadyClaimerSkipsExternallyBlockedWork(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := newRemoteStore(a, b) + blockOnPayments(raw, a.ID) + claimer, err := testStore(raw, &fakeStore{}, true).ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer: %v", err) + } + + next, err := claimer.ClaimNext(t.Context(), publicops.ClaimNextRequest{Actor: "worker"}) + if err != nil { + t.Fatalf("ClaimNext: %v", err) + } + if next.Claimed == nil || next.Claimed.ID != b.ID || b.Assignee != "worker" { + t.Fatalf("ClaimNext claimed %+v, want %s for worker", next.Claimed, b.ID) + } + if want := []string{"Claim be-b"}; !slices.Equal(raw.served, want) { + t.Fatalf("served calls = %v, want %v", raw.served, want) + } +} + +// TestRemoteReadyClaimerReportsLostRaces: a front that had work but lost every +// candidate is an error, not the empty answer that means there is no work. +func TestRemoteReadyClaimerReportsLostRaces(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := newRemoteStore(a, b) + blockOnPayments(raw, a.ID) + raw.claimErr = storage.ErrAlreadyClaimed + claimer, err := testStore(raw, &fakeStore{}, true).ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer: %v", err) + } + + next, err := claimer.ClaimNext(t.Context(), publicops.ClaimNextRequest{Actor: "worker"}) + if err == nil || !strings.Contains(err.Error(), "lost 1 races") || next.Claimed != nil { + t.Fatalf("ClaimNext = %+v, %v; want a lost-races error", next.Claimed, err) + } +} + +// TestRemoteBlockingAnnotatorMergesExternalBlockers: the served annotation's +// own blockers stay, the external ones join them, and the edges behind them +// are read in chunks the wire accepts. +func TestRemoteBlockingAnnotatorMergesExternalBlockers(t *testing.T) { + issues := make([]*types.Issue, 0, 150) + ids := make([]string, 0, 150) + for i := range 150 { + issues = append(issues, issue(fmt.Sprintf("be-%03d", i))) + ids = append(ids, issues[i].ID) + } + raw := newRemoteStore(issues...) + blockOnPayments(raw, "be-000", "be-149") + raw.localBlockers = map[string][]string{"be-000": {"be-local"}} + annotator, err := testStore(raw, &fakeStore{}, true).BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator: %v", err) + } + + result, err := annotator.AnnotateBlocking(t.Context(), publicops.BlockingRequest{IDs: ids}) + if err != nil { + t.Fatalf("AnnotateBlocking: %v", err) + } + blockedBy := make(map[string][]string, len(result.Items)) + for _, item := range result.Items { + blockedBy[item.ID] = item.BlockedBy + } + if want := []string{"be-local", paymentsRef}; !slices.Equal(blockedBy["be-000"], want) { + t.Errorf("be-000 blocked by %v, want %v", blockedBy["be-000"], want) + } + if want := []string{paymentsRef}; !slices.Equal(blockedBy["be-149"], want) { + t.Errorf("be-149 blocked by %v, want %v", blockedBy["be-149"], want) + } + if len(blockedBy["be-001"]) != 0 { + t.Errorf("be-001 blocked by %v, want nothing", blockedBy["be-001"]) + } + if want := []int{100, 50}; !slices.Equal(raw.edgeReads, want) { + t.Errorf("served edge reads named %v anchors, want %v", raw.edgeReads, want) + } +} + +// TestRemoteTreeWalkerAddsExternalLeaves: the served down-tree gains the leaf +// its stored edges name, a leaf the server rendered is not repeated, and any +// other walk is the served answer as it came. +func TestRemoteTreeWalkerAddsExternalLeaves(t *testing.T) { + root := &types.TreeNode{Issue: *issue("be-a")} + served := &types.TreeNode{Issue: types.Issue{ID: paymentsRef, Title: "served leaf"}, Depth: 1, ParentID: "be-a"} + for _, tc := range []struct { + name string + req publicops.WalkTreeRequest + tree []*types.TreeNode + wantIDs []string + }{ + {name: "down walk", req: publicops.WalkTreeRequest{RootID: "be-a", MaxDepth: 50}, tree: []*types.TreeNode{root}, wantIDs: []string{"be-a", paymentsRef}}, + {name: "server rendered the leaf", req: publicops.WalkTreeRequest{RootID: "be-a", MaxDepth: 50}, tree: []*types.TreeNode{root, served}, wantIDs: []string{"be-a", paymentsRef}}, + {name: "up walk", req: publicops.WalkTreeRequest{RootID: "be-a", MaxDepth: 50, Direction: publicops.TreeUp}, tree: []*types.TreeNode{root}, wantIDs: []string{"be-a"}}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := newRemoteStore() + raw.tree = tc.tree + blockOnPayments(raw, "be-a") + walker, err := testStore(raw, &fakeStore{}, true).TreeWalker() + if err != nil { + t.Fatalf("TreeWalker: %v", err) + } + + result, err := walker.WalkTree(t.Context(), tc.req) + if err != nil { + t.Fatalf("WalkTree: %v", err) + } + ids := make([]string, 0, len(result.Nodes)) + for _, node := range result.Nodes { + ids = append(ids, node.ID) + } + if !slices.Equal(ids, tc.wantIDs) { + t.Fatalf("tree = %v, want %v", ids, tc.wantIDs) + } + if leaf := result.Nodes[len(result.Nodes)-1]; tc.name == "down walk" { + if leaf.Status != types.StatusOpen || leaf.ParentID != "be-a" || leaf.Depth != 1 || leaf.Title != "○ payments" { + t.Fatalf("leaf = %+v, want the open payments leaf under be-a", leaf) + } + } + }) + } +} + +// TestRemoteBatchCloserRefusesExternallyBlockedItems: a still-open externally +// blocked item is refused the way the local policy close refuses it and the +// rest close in one served request. An item already closed keeps the served +// answer, Force waives the policy, and a server that enforces the policy gets +// the whole batch. +func TestRemoteBatchCloserRefusesExternallyBlockedItems(t *testing.T) { + for _, tc := range []struct { + name string + force bool + enforced bool + wantRefused bool + }{ + {name: "policy", wantRefused: true}, + {name: "force", force: true}, + {name: "server enforces", enforced: true}, + } { + t.Run(tc.name, func(t *testing.T) { + blocked, free, done := issue("be-blocked"), issue("be-free"), issue("be-done") + done.Status = types.StatusClosed + raw := newRemoteStore(blocked, free, done) + raw.enforced = tc.enforced + blockOnPayments(raw, blocked.ID, done.ID) + closer, err := testStore(raw, &fakeStore{}, true).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser: %v", err) + } + + request := publicops.CloseBatchRequest{Actor: "worker", Force: tc.force, Items: closeItems(blocked.ID, free.ID, done.ID)} + result, err := closer.CloseBatch(t.Context(), request) + if err != nil { + t.Fatalf("CloseBatch: %v", err) + } + if len(result.Outcomes) != len(request.Items) { + t.Fatalf("CloseBatch answered %d outcomes for %d items", len(result.Outcomes), len(request.Items)) + } + for i, outcome := range result.Outcomes { + if outcome.IssueID != request.Items[i].IssueID { + t.Fatalf("outcome %d is %s's, want %s's", i, outcome.IssueID, request.Items[i].IssueID) + } + } + wantServed := []string{blocked.ID, free.ID, done.ID} + if tc.wantRefused { + refusal := result.Outcomes[0].Err + want := "cannot close blocked issue: be-blocked is blocked by [" + paymentsRef + "]" + if !errors.Is(refusal, storage.ErrCloseBlocked) || refusal.Error() != want { + t.Fatalf("be-blocked outcome = %v, want %q", refusal, want) + } + wantServed = []string{free.ID, done.ID} + } else if result.Outcomes[0].Err != nil { + t.Fatalf("be-blocked outcome = %v, want it closed", result.Outcomes[0].Err) + } + if result.Outcomes[1].Err != nil || result.Outcomes[2].Err != nil { + t.Fatalf("be-free, be-done outcomes = %v, %v; want the served answers", result.Outcomes[1].Err, result.Outcomes[2].Err) + } + if want := [][]string{wantServed}; !slices.EqualFunc(raw.closes, want, slices.Equal[[]string]) { + t.Fatalf("served closes = %v, want %v", raw.closes, want) + } + }) + } +} + +// TestRemoteBatchCloserRefusesANextClaim: the served batch close cannot carry +// a next claim, so a batch with one is refused whole, the same way whether the +// workspace holds an open holder of an external ref, only a closed one, or +// none, and with Force too. The refusal leads with that wire fact, reads +// nothing, closes nothing and names the commands that do the same work. A +// server that enforces the policy itself gets the request as it came, and its +// closer answers. +func TestRemoteBatchCloserRefusesANextClaim(t *testing.T) { + for _, tc := range []struct { + name string + holder types.Status + force bool + enforced bool + wantServed []string + }{ + {name: "open holder", holder: types.StatusOpen}, + {name: "closed holder", holder: types.StatusClosed}, + {name: "no holder"}, + {name: "force", holder: types.StatusOpen, force: true}, + {name: "server enforces", holder: types.StatusOpen, enforced: true, wantServed: []string{"CloseBatch ClaimNext"}}, + } { + t.Run(tc.name, func(t *testing.T) { + free, holder := issue("be-free"), issue("be-holder") + raw := newRemoteStore(free, holder) + raw.enforced = tc.enforced + if tc.holder != "" { + holder.Status = tc.holder + blockOnPayments(raw, holder.ID) + } + closer, err := testStore(raw, &fakeStore{}, true).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser: %v", err) + } + + _, err = closer.CloseBatch(t.Context(), publicops.CloseBatchRequest{ + Actor: "worker", Force: tc.force, Items: closeItems(free.ID), ClaimNext: &publicops.ReadyRequest{}, + }) + if tc.enforced { + if !errors.Is(err, errServedClaimNextRefused) { + t.Fatalf("CloseBatch with ClaimNext = %v, want the served closer's answer", err) + } + } else { + var unsupported *storage.ErrUnsupported + if !errors.As(err, &unsupported) || unsupported.Op != "CloseBatchRequest.ClaimNext" { + t.Fatalf("CloseBatch with ClaimNext = %v, want the CloseBatchRequest.ClaimNext refusal", err) + } + if msg := err.Error(); !strings.Contains(msg, "cannot carry a next claim") || !strings.Contains(msg, "nothing was closed") || !strings.Contains(msg, "then run `bd ready --claim`") { + t.Fatalf("refusal %q does not say why, that nothing closed, and what to run instead", msg) + } + } + if !slices.Equal(raw.served, tc.wantServed) { + t.Fatalf("served calls = %v, want %v", raw.served, tc.wantServed) + } + if len(raw.closes) != 0 || len(raw.edgeReads) != 0 || raw.scans != 0 { + t.Fatalf("closes %v, edge reads %v, scans %d; want none", raw.closes, raw.edgeReads, raw.scans) + } + }) + } +} + +// TestRemoteBatchCloserValidatesANextClaimFirst: a next claim that is invalid +// on every backend is ErrValidation, not the refusal. +func TestRemoteBatchCloserValidatesANextClaimFirst(t *testing.T) { + for name, claim := range map[string]*publicops.ReadyRequest{ + "offset": {Offset: 1}, + "bad sort": {Sort: "bogus"}, + } { + t.Run(name, func(t *testing.T) { + raw := newRemoteStore(issue("be-free")) + closer, err := testStore(raw, &fakeStore{}, true).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser: %v", err) + } + + _, err = closer.CloseBatch(t.Context(), publicops.CloseBatchRequest{ + Actor: "worker", Items: closeItems("be-free"), ClaimNext: claim, + }) + var unsupported *storage.ErrUnsupported + if !errors.Is(err, storage.ErrValidation) || errors.As(err, &unsupported) { + t.Fatalf("CloseBatch with ClaimNext %+v = %v, want ErrValidation", claim, err) + } + }) + } +} diff --git a/internal/storage/externaldeps/role_accessors.go b/internal/storage/externaldeps/role_accessors.go index 464bf30dc..7ba88001d 100644 --- a/internal/storage/externaldeps/role_accessors.go +++ b/internal/storage/externaldeps/role_accessors.go @@ -14,8 +14,19 @@ import ( // IssueReader builds the read role on the policy store. In particular, Ready // and List(ReadyFlag) must call this store's filtered ready methods rather than -// promoted methods on the undecorated store. -func (s *Store) IssueReader() (issueops.Reader, error) { return storereader.New(s) } +// promoted methods on the undecorated store. Over a remote store the served +// reader answers everything else (remote_roles.go). +func (s *Store) IssueReader() (issueops.Reader, error) { + policy, err := storereader.New(s) + if err != nil || !s.servesRoles() { + return policy, err + } + served, err := s.inner.IssueReader() + if err != nil { + return nil, err + } + return &remoteReader{served: served, policy: policy}, nil +} // IssueClaimer rejects a direct claim of externally blocked work before the // backend's atomic claim operation. ReadyClaimer below handles selection among @@ -25,6 +36,9 @@ func (s *Store) IssueClaimer() (issueops.Claimer, error) { if err != nil { return nil, err } + if s.servesRoles() { + return &remoteClaimer{served: inner, policy: s}, nil + } return &issueClaimer{inner: inner, policy: s}, nil } @@ -47,6 +61,9 @@ func (c *issueClaimer) Claim(ctx context.Context, req issueops.ClaimRequest) (is // ReadyClaimer keeps external blockers out of the ready-claim selection used // by HTTP serving. The local compare-and-swap remains inside ClaimReadyIssue. func (s *Store) ReadyClaimer() (issueops.ReadyClaimer, error) { + if s.servesRoles() { + return &remoteReadyClaimer{policy: s}, nil + } return &readyClaimer{policy: s}, nil } @@ -86,6 +103,9 @@ func (s *Store) BlockingAnnotator() (issueops.BlockingAnnotator, error) { if err != nil { return nil, err } + if s.servesRoles() { + return &remoteBlockingAnnotator{served: inner, policy: s}, nil + } return &blockingAnnotator{inner: inner, policy: s}, nil } @@ -119,6 +139,9 @@ func (s *Store) TreeWalker() (issueops.TreeWalker, error) { if err != nil { return nil, err } + if s.servesRoles() { + return &remoteTreeWalker{served: inner, policy: s}, nil + } return &treeWalker{inner: inner, policy: s}, nil } @@ -128,7 +151,7 @@ type treeWalker struct { } func (t *treeWalker) WalkTree(ctx context.Context, req issueops.WalkTreeRequest) (issueops.TreeResult, error) { - if (req.Direction != "" && req.Direction != issueops.TreeDown) || req.Status != "" || req.MaxRows != 0 { + if !plainDownTree(req) { return t.inner.WalkTree(ctx, req) } nodes, err := t.policy.GetDependencyTree(ctx, req.RootID, req.MaxDepth, false, false) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 61f80cde0..c4a2ba951 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -1022,6 +1022,55 @@ type ExternalDependencyQueryStore interface { GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) } +// RemoteBackendStore is implemented by a DoltStorage that is a pure network +// client of a remote bd serve process (a registered backend whose +// backends.Backend.Remote is true — see internal/storage/backends). It is +// metadata about the store's transport, not a policy decision. Its one in-tree +// consumer is the external-deps decorator, which wraps a remote store like any +// other and reads this marker only to choose how it composes its roles: over a +// remote store it builds each role on the store's own served role instead of +// the legacy methods such a store refuses. Whether the decorator skips its +// client-side enforcement is ExternalDependencyPolicyProber's question. +type RemoteBackendStore interface { + IsRemoteBackendStore() bool +} + +// ExternalDependencyPolicyProber is implemented by a DoltStorage that can +// answer whether the remote server it talks to already enforces bd's +// external-dependency policy itself (design 3.6, "External-dependency server +// policy": upstream's storage.ServerEnforcedPolicy / PolicyEnforcedByServer). +// The external-deps decorator consults this — never RemoteBackendStore, which +// only shapes how it composes its roles — before skipping its own client-side +// enforcement: a remote store whose +// server advertises the capability (httpapi's policy.external_dependencies) +// has already enforced the policy before answering, so a second client-side +// pass would be redundant. A store that does not implement this interface, or +// that implements it and reports false — including a remote store whose +// server is silent on the capability — gets the ordinary client-side +// enforcement. The policy is never silently skipped merely because the store +// is remote. +type ExternalDependencyPolicyProber interface { + ServerEnforcesExternalDependencyPolicy(ctx context.Context) (bool, error) +} + +// ExcludeIDsUnsupportedStore is implemented by a DoltStorage whose ready-work +// reads cannot express types.WorkFilter.ExcludeIDs over their own transport. +// Today that is exactly httpclient.Store: the v0 wire's listReadyWork and +// countReadyWork operations publish no id-exclusion parameter, so the http +// bridge refuses rather than silently widening the result set whenever a +// filter carries any (design 3.6 / L12, "every field it cannot express +// refuses"). +// +// The external-deps decorator consults this before deciding how to apply its +// OWN additional exclusions (issues blocked by an unsatisfied +// external:: dependency): a store that answers true here +// gets those exclusions applied client-side in Go instead of folded into the +// filter handed down, so the policy still runs — design 3.6 says it is never +// silently skipped — without tripping the wire's own refusal. +type ExcludeIDsUnsupportedStore interface { + ExcludeIDsUnsupported() bool +} + // Transaction provides atomic multi-operation support within a single database transaction. // // The Transaction interface exposes a subset of storage methods that execute within diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index c2e206425..ebbedeaa1 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -898,6 +898,10 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { []string{"$(rootpath :served_pinned_shards.txt)", "$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1"}, }, + "//backend/http:http_served_test": { + []string{"$(rootpath :http_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, + map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, "//cmd/bd:bd_proxied_test": { []string{"$(rootpath //:.github/scripts/proxied-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)"}, map[string]string{ diff --git a/test/embedder/BUILD.bazel b/test/embedder/BUILD.bazel new file mode 100644 index 000000000..f17f9cd85 --- /dev/null +++ b/test/embedder/BUILD.bazel @@ -0,0 +1,73 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "embedder_test", + srcs = ["embedder_test.go"], + deps = [ + "//:beads", + "//backend", + "//backend/http", + "//issueops", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/test/embedder/embedder_test.go b/test/embedder/embedder_test.go new file mode 100644 index 000000000..66ab6c6c5 --- /dev/null +++ b/test/embedder/embedder_test.go @@ -0,0 +1,253 @@ +// Package embedder_test proves the one promise S6 makes to an out-of-tree +// embedder (gc, Gas City is the motivating case): linking nothing but PUBLIC +// doors — github.com/steveyegge/beads, .../backend, and .../backend/http — is +// enough to register the http backend and open a workspace through it with a +// per-call credential, via beads.OpenBestAvailableWith. +// +// Written fresh for OSS beads S6 (no bd-enterprise source copied). This +// package imports ONLY those three import paths (plus the standard library +// and net/http/httptest for the stand-in server): no internal/... package +// appears in this file's imports, which is itself the thing under test — an +// internal import here would compile today and break silently the day +// internal/httpclient's shape changes, defeating the whole point of a public +// door. +package embedder_test + +import ( + "context" + "encoding/json" + "go/parser" + "go/token" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + + beads "github.com/steveyegge/beads" + "github.com/steveyegge/beads/backend" + bdhttp "github.com/steveyegge/beads/backend/http" + "github.com/steveyegge/beads/issueops" +) + +// TestNoInternalImports is the mechanical, enforced form of this file's doc +// comment promise above: nothing in this package may import a +// github.com/steveyegge/beads/internal/... package, directly or otherwise. +// That import would compile fine today and only break silently the day some +// internal package's shape changes — exactly what a public-door embedder +// test exists to catch before it ships. It replaces an earlier, separate +// go.mod for this directory: a second module cannot see internal/... either, +// but it is heavier machinery for the same guarantee, and nothing stops a +// future edit from deleting a go.mod quietly, whereas this test fails loudly +// in the same `go test` run as everything else. +// +// It only inspects this package's own files' import declarations +// (go/parser, ImportsOnly — no type-checking, so it stays fast and adds no +// dependency beyond the standard library) and does not recurse into what +// those imports themselves pull in; that transitive check is exactly what +// go build/go vet already do on every change. +func TestNoInternalImports(t *testing.T) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", nil, parser.ImportsOnly) + if err != nil { + t.Fatalf("ParseDir: %v", err) + } + const forbidden = "github.com/steveyegge/beads/internal" + for _, pkg := range pkgs { + for filename, file := range pkg.Files { + for _, imp := range file.Imports { + path, err := strconv.Unquote(imp.Path.Value) + if err != nil { + t.Fatalf("%s: unquoting import %s: %v", filename, imp.Path.Value, err) + } + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + t.Errorf("%s imports %q: test/embedder must link only public doors, never an internal/... package", filename, path) + } + } + } + } +} + +const embedderProjectID = "proj-gc-tenant" + +// embedderReadyIssueID is the one issue the stand-in server's /v0/beads/ready +// page carries, so the test has something distinctive to assert on besides +// "the call did not error". +const embedderReadyIssueID = "gc-1" + +// backendName is spelled literally, the way a workspace's metadata.json +// carries it, rather than read back from a re-exported constant: bdhttp +// deliberately exports no Backend name constant (see backend/http/http.go), +// so a rename of the registry key here would be caught by every CONNECTED +// workspace left unopenable, which is the fact this spells out explicitly +// instead of hiding behind a symbol that does not exist. +const backendName = "http" + +// TestEmbedderOpensTheHTTPBackendThroughThePublicDoorsOnly is the linkage +// proof: Register from the public backend/http door, a workspace described +// only by the public Target/SaveTarget helpers, and +// beads.OpenBestAvailableWith from the public root package — no package under +// internal/... in the call chain this test writes. +func TestEmbedderOpensTheHTTPBackendThroughThePublicDoorsOnly(t *testing.T) { + server := newStandInServer(t) + bdhttp.Register(bdhttp.Options{HTTPClient: server.Client()}) + t.Cleanup(func() { backend.Deregister(backendName) }) + + beadsDir := filepath.Join(t.TempDir(), ".beads") + // SaveTarget, like the package's other low-level helpers, writes into an + // already-existing directory; creating it is the caller's job (cmd/bd's + // connect command does the same MkdirAll before calling SaveTarget). + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("mkdir %s: %v", beadsDir, err) + } + base, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("parse stand-in server URL: %v", err) + } + target := bdhttp.Target{BaseURL: base, ExpectProjectID: embedderProjectID} + // Attach is the one public call that activates a workspace for this + // backend: it writes the per-user sidecar (what SaveTarget alone used to + // leave to this test) AND metadata.json's {"backend": "http"} selection — + // the fact that tells OpenBestAvailableWith to consult the registry at + // all — in one call, the same pair `bd connect` itself now calls through + // (cmd/bd/connect.go). Before Attach existed this test hand-wrote + // metadata.json's JSON shape directly, since there was no public door + // onto that second file's writer; asserting the on-disk shape below + // keeps this test proving the SAME canary (a rename of the registry key + // breaks every connected workspace) without reaching into + // internal/configfile for it. + if err := bdhttp.Attach(beadsDir, target); err != nil { + t.Fatalf("Attach: %v", err) + } + metadataBytes, err := os.ReadFile(filepath.Join(beadsDir, "metadata.json")) + if err != nil { + t.Fatalf("reading metadata.json after Attach: %v", err) + } + var metadata struct { + Backend string `json:"backend"` + } + if err := json.Unmarshal(metadataBytes, &metadata); err != nil { + t.Fatalf("parsing metadata.json after Attach: %v", err) + } + if metadata.Backend != backendName { + t.Fatalf("metadata.json backend = %q, want %q", metadata.Backend, backendName) + } + + // A multi-tenant embedder's reason to call OpenBestAvailableWith rather + // than OpenBestAvailable: this credential is THIS caller's, not whatever + // the ambient process environment happens to hold, which belongs to no + // one tenant in a process serving many. + provider := &tenantCredential{token: "gc-tenant-token"} + store, err := beads.OpenBestAvailableWith(context.Background(), beadsDir, beads.OpenOptions{ + Credential: bdhttp.ProvidedCredential{Provider: provider}, + }) + if err != nil { + t.Fatalf("OpenBestAvailableWith: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + // The read itself goes through the role accessor, not a backend-specific + // method: IssueReader().Ready is issueops.Reader's own door, the same one + // `bd ready` calls against ANY backend. Ready is the simplest operation to + // stand a fake server in for — a single GET with no pagination contract + // and no handshake capability gate (it is a baseline v0 operation) — so + // this proves the public linkage without reimplementing the wire's + // capability negotiation in a throwaway test server. + reader, err := store.IssueReader() + if err != nil { + t.Fatalf("IssueReader: %v", err) + } + page, err := reader.Ready(context.Background(), issueops.ReadyRequest{}) + if err != nil { + t.Fatalf("Ready: %v", err) + } + if len(page.Items) != 1 || page.Items[0].ID != embedderReadyIssueID { + t.Errorf("ready page = %+v, want one item %q", page.Items, embedderReadyIssueID) + } + server.wantAuthorization(t, "Bearer gc-tenant-token") +} + +// tenantCredential is a bdhttp.CredentialProvider scoped to one embedded +// tenant, built from nothing but the public CredentialProvider interface. +type tenantCredential struct{ token string } + +func (c *tenantCredential) Authorize(_ context.Context, req *http.Request) error { + req.Header.Set("Authorization", "Bearer "+c.token) + return nil +} + +func (c *tenantCredential) Refresh(context.Context) (bool, error) { return false, nil } + +// standInServer is a minimal bd serve stand-in: just enough of the v0 +// handshake response for Open/GetMetadata to succeed, recording the +// Authorization header every request carried. +type standInServer struct { + *httptest.Server + mu sync.Mutex + auths []string +} + +func newStandInServer(t *testing.T) *standInServer { + t.Helper() + s := &standInServer{} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.auths = append(s.auths, r.Header.Get("Authorization")) + s.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v0/beads/context": + // Open/OpenWith dial lazily and never fetch this on their own + // (the handshake is cached per-process on the first POST-BASELINE + // dispatch — see internal/httpclient.Store.snapshot) — Ready + // below is baseline and preflights without one. This handler + // stays in place anyway: it is the one other request a workspace + // opened through this backend could legitimately make, and a + // stand-in server that 404s it would be a trap for the next + // person who extends this test to a non-baseline operation. + _ = json.NewEncoder(w).Encode(map[string]any{ + "api_version": "v0", + "backend": "dolt", + "bd_version": "9.9.9", + "beads_dir": "/srv/.beads", + "capabilities": []string{"issues.get", "issues.list"}, + "database": "beads", + "dolt_mode": "server", + "project_id": embedderProjectID, + "repo_root": "/srv", + "schema_version": 1, + "wire_revision": 1, + "min_client_wire_revision": 1, + }) + case "/v0/beads/ready": + _ = json.NewEncoder(w).Encode(map[string]any{ + "has_more": false, + "items": []map[string]any{ + {"id": embedderReadyIssueID, "title": "embedder linkage proof", "priority": 2}, + }, + }) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(s.Server.Close) + return s +} + +func (s *standInServer) wantAuthorization(t *testing.T, want string) { + t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + if len(s.auths) == 0 { + t.Fatal("the stand-in server saw no requests at all") + } + for i, got := range s.auths { + if got != want { + t.Errorf("request %d carried Authorization %q, want %q", i, got, want) + } + } +} diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index fcddec75c..b7e0b6570 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -177,6 +177,7 @@ release_cross_build( ":pure_bd_has_no_cgo_only_deps", "//:beads", "//backend/conformance:conformance", + "//backend/http:http", "//backend:backend", "//beadserrors:beadserrors", "//cmd/bd/doctor/fix:fix", From eb204363c953d9eaabe7f18f7a6d313e0f21ffcb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 10:13:49 +0000 Subject: [PATCH 035/151] ci(bazel): add advisory dolt-race shadow lane (item 5 phase 1) (#7464) * ci(bazel): add advisory dolt-race shadow lane (item 5 phase 1) Add --config=dolt-race, the union of --config=doltserver and --config=doltserver-proxied with one tag filter for both tiers, and an advisory bazel-dolt-race job that runs it beside bazel-doltserver and bazel-proxied. The shadow measures one client per group of lanes against the two lanes it would replace before any cutover (item 5 slice A1a). The job runs in mode remote on pull_request and merge_group events only. It has job-level continue-on-error, no job or workflow_call output, and no gate names it, so it cannot change CI Gate or PR Risk Gate. Its ci-analytics step is copied verbatim with LANE=dolt-race, and rrc seeds the new config. Policy tests pin the config as the exact union of its members' lines (tag filter aside), the job's run modes and events, its advisory shape, the analytics lane map and the rrc command list. aquery shows every member target's TestRunner action (actionKey, argv, env, inputs, outputs) is unchanged under the union config. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): give the dolt-race shadow timeout headroom; fix gate-wait wording Review follow-up on the advisory bazel-dolt-race job (maintainer-city-c3, findings 1, 2 and 5): - Raise the job's timeout-minutes from 30 to 35. continue-on-error covers a failed or timed-out step, but not a job GitHub cancels at its own timeout, which could make the bazel call non-success and turn CI Gate red. TestBazelDoltRaceShadowIsAdvisory now reads both timeouts from the workflow and requires job >= test step + 10 (and a step timeout at all). - Correct the docs and comments: the shadow cannot fail the gate, but CI Gate still waits for it, because its needs on the bazel call completes only when every job in it has finished. - Correct the checks comment: check_testcases covers every target in the union, including the dolt-server tier its own lane never checked. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): bound every dolt-race shadow step; job timeout covers their sum Review follow-up (S1): a hang in any bazel-dolt-race step without a timeout would run into the job's own timeout, and a job GitHub cancels there is not covered by continue-on-error. Every step now has a timeout-minutes, and the job's timeout covers their sum: - checkout 5, rrc read 1, setup-bazel 10, flaky query 5, checks 3 each, test-log upload 3. The test step (25) and the analytics steps (3, 2) keep bazel-proxied's values; bazel-proxied sets no timeout on the others, so these bound them. - The step timeouts sum to 60 minutes, so the job timeout goes from 35 to 65. - TestBazelDoltRaceShadowIsAdvisory now requires a timeout on every step and job >= sum(step timeouts) + 5, read from the workflow, alongside the existing job >= test step + 10. Also cite the farm maintainer's confirmation of NativeLink 1.7.1's in-flight Execute dedupe in the job comment. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): keep dolt-race shadow timeout under the merge queue's Tighten the shadow's non-test step budgets so the job timeout (step sum + 5) is 55 minutes, and pin that it stays below the 60-minute merge queue check timeout: CI Gate waits for the shadow, so a hung shadow must not drop a group from the queue. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 26 +++ .github/scripts/rrc-lane-commands.txt | 1 + .github/workflows/bazel.yml | 173 +++++++++++++- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 33 +++ scripts/BUILD.bazel | 1 + scripts/bazel_cache_mode_test.go | 4 +- scripts/bazel_dolt_race_shadow_test.go | 305 +++++++++++++++++++++++++ scripts/bazel_rrc_test.go | 5 +- scripts/ci_analytics_workflow_test.go | 3 + scripts/ci_merge_queue_test.go | 1 + scripts/ci_workflow_test.go | 36 ++- scripts/pr_risk_bazel_coverage_test.go | 3 + 12 files changed, 584 insertions(+), 7 deletions(-) create mode 100644 scripts/bazel_dolt_race_shadow_test.go diff --git a/.bazelrc b/.bazelrc index 63d000368..61e7e0207 100644 --- a/.bazelrc +++ b/.bazelrc @@ -161,6 +161,32 @@ test:doltserver-proxied --remote_download_regex=.*/test\.(log|xml)$ # visible "Re-run failed jobs" retries it. test:doltserver-proxied --experimental_remote_cache_eviction_retries=0 +# dolt-server + proxied union (item 5, Phase 1 shadow): bazel.yml's advisory +# bazel-dolt-race job runs both tiers in one invocation, beside their own +# lanes, to measure one client per group of lanes before any cutover. Exactly +# the union of --config=doltserver's and --config=doltserver-proxied's lines, +# with one tag filter naming both tiers +# (scripts/bazel_dolt_race_shadow_test.go). Every other flag is the same in +# both members or outside any action key for the other: the race build, +# -test.parallel=4 and GO_TEST_WRAP_TESTV=1 are common; the eternal timeout +# applies to no dolt-server target; the eviction retry count is what +# doltserver already gets from --config=sole-run in mode remote, the only mode +# the shadow runs in; test_summary, local_test_jobs and remote_download_regex +# are not in any action key. So each member target's test action is +# unchanged, and the two lanes' cached results carry over. The member configs +# stay for make, developers and the member lanes. +test:dolt-race --@rules_go//go/config:race +test:dolt-race --test_tag_filters=dolt-server,dolt-server-proxied +test:dolt-race --build_tests_only +test:dolt-race --keep_going +test:dolt-race --test_summary=terse +test:dolt-race --test_timeout=-1,-1,-1,1200 +test:dolt-race --test_env=GO_TEST_WRAP_TESTV=1 +test:dolt-race --test_arg=-test.parallel=4 +test:dolt-race --local_test_jobs=4 +test:dolt-race --remote_download_regex=.*/test\.(log|xml)$ +test:dolt-race --experimental_remote_cache_eviction_retries=0 + # Server-Dolt storage tier: PR Risk's "Test (Server Dolt Conformance)" and # "Test (Server Dolt Full Suite N/16)" jobs, as # //internal/storage/dolt:dolt_server_{conformance,full}_test diff --git a/.github/scripts/rrc-lane-commands.txt b/.github/scripts/rrc-lane-commands.txt index 08bd7eb5e..8821ad271 100644 --- a/.github/scripts/rrc-lane-commands.txt +++ b/.github/scripts/rrc-lane-commands.txt @@ -9,6 +9,7 @@ test //... --config=embedded test //... --config=integration test //... --config=doltserver test //... --config=doltserver-proxied +test //... --config=dolt-race test //... --config=doltserver-integration test //... --config=doltserver-cmd build --config=pure //cmd/bd:bd //cmd/bd:bd_test //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index ce9bf223b..e78832963 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -46,7 +46,11 @@ # only what the PR changed is built and tested on the runner, and it is # their only PR-time integration run (no legacy job runs it on PRs). Mode # local (rbe=off) still skips it. Direct runs (push to main, -# workflow_dispatch, nightly.yml's call) gate nothing. +# workflow_dispatch, nightly.yml's call) gate nothing. bazel-dolt-race (the +# dolt-server + proxied union shadow, mode remote on PRs and merge groups +# only) cannot fail the gate either: no output, job-level continue-on-error. +# The gate still waits for it, as for every job of the call (see its comment +# and engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "dolt-race Shadow"). # # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to @@ -1983,6 +1987,173 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # Item 5, Phase 1 (scripts/bazel_dolt_race_shadow_test.go): the advisory + # shadow of a union lane. One `bazel test --config=dolt-race` runs the + # dolt-server and proxied-server tiers together (.bazelrc: exactly the + # union of --config=doltserver's and --config=doltserver-proxied's lines, + # one tag filter for both), beside bazel-doltserver and bazel-proxied, so + # one client per group of lanes can be compared with the two it would + # replace (end time, runner minutes, test counts and results) before any + # cutover. Each member target's test action key is unchanged, so the shadow + # and the member lanes share cached results; and rbe-west deduplicates + # identical in-flight executions (confirmed by the farm maintainer: + # NativeLink 1.7.1 subscribes a second identical Execute of a cacheable + # action to the operation already in flight), so the shadow adds no + # executions. + # Advisory: it cannot fail the gate. Job-level continue-on-error (a failed + # job otherwise fails the whole call, whose result pr.yml's ci-gate + # requires as BAZEL), no workflow_call output, no job outputs, no job in + # this file needs it, and no gate script or caller names it. The gate does + # wait for it: ci-gate needs pr.yml's bazel call, which completes only when + # every job in it has finished (success, failure or timeout), so a slow + # shadow delays the gate and the merge queue. continue-on-error does not + # cover a job GitHub cancels at its own timeout-minutes, which could turn + # the call red: every step has a timeout, and the job's exceeds their sum + # (TestBazelDoltRaceShadowIsAdvisory). Mode remote only, on + # same-repo pull requests and merge groups (mode remote on a pull_request + # is never a fork): no fork mode, no cache or local mode, and not on push, + # nightly, dispatch or bazel-farm.yml's call. Its ci-analytics LANE is + # dolt-race. Removing it is the rollback; nothing reads it. + bazel-dolt-race: + name: dolt-race + needs: rbe + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.mode == 'remote' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} + # At least every step's timeout-minutes plus 5 (and the test step's plus + # 10): a hung step then fails the job (continue-on-error) before GitHub + # cancels it, which continue-on-error does not cover + # (TestBazelDoltRaceShadowIsAdvisory). bazel-proxied's steps have no + # timeouts but its test step's; these bound the rest. + timeout-minutes: 55 + continue-on-error: true + steps: + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 5 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + timeout-minutes: 1 + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + + - name: Set up Bazel + id: bazel + timeout-minutes: 5 + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} + + # As in bazel-proxied. + - name: No test target is marked flaky + timeout-minutes: 3 + run: | + flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" + if [ -n "$flaky" ]; then + echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" + exit 1 + fi + + # Both tiers' shards at once: 27 dolt-server + 35 proxied test actions, + # under the remote-exec config's 64 jobs. The step timeout is + # bazel-proxied's (the longer tier on rbe-west). + - name: bazel test //... --config=dolt-race + id: test + timeout-minutes: 25 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=dolt-race ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=dolt-race: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # The member lanes' checks over the union's BEP: bazel-proxied's (the + # dolt-server lane has none). check_shard_coverage ignores targets it + # does not list; check_testcases checks every target, so it covers the + # dolt-server tier too, which bazel-doltserver does not check: a + # dolt-server shard that runs no test turns this job red while both + # member lanes stay green. + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + timeout-minutes: 2 + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + + - name: Every listed test ran in its shard + if: ${{ always() && steps.test.outcome != 'skipped' }} + timeout-minutes: 2 + run: >- + python3 tools/bazel/check_shard_coverage.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 34 + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + timeout-minutes: 3 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: bazel-dolt-race-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 2 + env: + LANE: dolt-race + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-dolt-race-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # pr-risk.yml's server-Dolt storage tier, "Test (Server Dolt Conformance)" # and "Test (Server Dolt Full Suite N/16)", as --config=doltserver-integration: # the conformance and 16-shard full-suite variants of diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index ce91016b0..6b6a98be0 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -620,6 +620,39 @@ job's `if:` or rotating the key, in either of two ways: step's own kill switch, checked before any network call. Deleting the variable does not disable pre-warming; it restores the default of 1. +## dolt-race Shadow (Not Required) + +`bazel.yml`'s `bazel-dolt-race` job (check name `Bazel / dolt-race`) is an +advisory shadow, not a required check. It runs +`bazel test //... --config=dolt-race`, the union of `--config=doltserver` and +`--config=doltserver-proxied` with one tag filter for both tiers, beside the +`bazel-doltserver` and `bazel-proxied` lanes. Its purpose is to measure one +client per group of lanes against the two lanes it would replace (end time, +runner minutes, test counts and results) before any cutover. Each member +target's test action key is unchanged, so the shadow shares cached results +with the member lanes. + +- It runs in mode `remote` on `pull_request` and `merge_group` events only. + It never runs in a fork mode (`fork-ro`, `fork-rw`), in mode `cache` or + `local`, on `bazel-farm.yml`'s call, on push, nightly or dispatch. +- It cannot fail the gate. It has job-level `continue-on-error`, so its + failure cannot fail the call's aggregate result (`BAZEL`). It has no + `workflow_call` output, no `bazel.yml` job `needs` it, and `pr.yml`, + `ci-gate.sh` and `bazel-gate.sh` never name it. +- The gate still waits for it. `CI Gate` needs `pr.yml`'s `bazel` call, and + a reusable-workflow call completes only when all of its jobs have finished + (success, failure or timeout), `continue-on-error` ones included. A slow + shadow therefore delays `CI Gate` and the merge queue, up to its + timeout. `continue-on-error` does not cover a job GitHub cancels at its + own `timeout-minutes`, so every step has its own timeout and the job's + (65 minutes) stays at least 5 minutes above their sum and 10 above the + test step's (policy-tested). A hung step then fails the job, which + `continue-on-error` covers, before GitHub would cancel it. +- Its ci-analytics `LANE` is `dolt-race`. +- Policy: `scripts/bazel_dolt_race_shadow_test.go`. Removing the job, its + `.bazelrc` config and its line in `.github/scripts/rrc-lane-commands.txt` + is the rollback; nothing reads them. + ## Remote Repo Contents Cache A cold lane client re-runs its repository rules (gazelle's `go_deps`, the Go diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index b3d3294a1..b5664d7a6 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -35,6 +35,7 @@ go_test( srcs = [ "bazel_autofix_push_test.go", "bazel_cache_mode_test.go", + "bazel_dolt_race_shadow_test.go", "bazel_farm_workflow_test.go", "bazel_fork_mode_test.go", "bazel_go_repository_tools_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 8212d4efd..74a0de735 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -225,7 +225,9 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { // no longer forwards the app secrets either, so there is no // audience left for pre-warming in fork-ro or fork-rw (see // bazel.yml's comment on the job). - case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && strings.HasPrefix(mode, "fork-")): + // Nor does a shadow lane (bazelShadowLanes): mode remote on + // same-repo PRs and merge groups only. + case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && strings.HasPrefix(mode, "fork-")) && bazelShadowLanes[name] == "": t.Errorf("%s does not run in mode %s (every lane runs remotely)", name, mode) case mode == "cache" && (name == bazelIntegJobName || name == bazelCmdDoltJobName) && !runs: t.Errorf("%s does not run in mode cache", name) diff --git a/scripts/bazel_dolt_race_shadow_test.go b/scripts/bazel_dolt_race_shadow_test.go new file mode 100644 index 000000000..0b6008047 --- /dev/null +++ b/scripts/bazel_dolt_race_shadow_test.go @@ -0,0 +1,305 @@ +package scripts_test + +import ( + "cmp" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// The dolt-race shadow (item 5, Phase 1): bazel.yml's bazel-dolt-race job +// runs --config=dolt-race, the union of --config=doltserver and +// --config=doltserver-proxied, in one `bazel test` beside the two member +// lanes, so the union-job model can be measured against them before any +// cutover. It must be advisory: no workflow_call output, nothing in pr.yml's +// gates or bazel-gate.sh/ci-gate.sh, and a failure that cannot fail the +// call's aggregate result (BAZEL) that pr.yml's ci-gate requires. It cannot +// fail the gate, but the gate still waits for it: ci-gate needs the whole +// bazel call, which completes only when every job in it has finished. + +const ( + bazelDoltRaceJobName = "bazel-dolt-race" + bazelDoltRaceConfig = "dolt-race" + // Mode remote only, and only on same-repo pull requests and merge + // groups (mode remote on a pull_request event is never a fork: forks + // get fork-ro, fork-rw or cache). Not on push, nightly, dispatch or + // bazel-farm.yml's pull_request_target call. + bazelDoltRaceJobIf = "${{ needs.rbe.outputs.mode == 'remote' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }}" +) + +// bazelShadowLanes: bazel.yml jobs that run beside gated lanes only to +// measure a change to them, and report into nothing (no job or +// workflow_call output, no gate id, job-level continue-on-error), with the +// reason. TestBazelLaneIsGated and the continue-on-error checks carve them +// out by this map. +var bazelShadowLanes = map[string]string{ + bazelDoltRaceJobName: "item 5 Phase 1: the dolt-server + proxied union beside its member lanes, advisory until a cutover PR", +} + +// bazelDoltRaceJobTimeoutHeadroom: the minutes the shadow job's +// timeout-minutes must exceed its test step's (TestBazelDoltRaceShadowIsAdvisory). +const bazelDoltRaceJobTimeoutHeadroom = 10 + +// bazelDoltRaceJobTimeoutMargin: the minutes the shadow job's timeout-minutes +// must exceed the sum of all its step timeouts. +const bazelDoltRaceJobTimeoutMargin = 5 + +// bazelDoltRaceMergeQueueTimeout: the merge queue's check timeout (ruleset +// merge_queue.check_response_timeout_minutes). CI Gate waits for the shadow, +// so the shadow's job timeout must stay below it or a hung shadow drops the +// group from the queue. +const bazelDoltRaceMergeQueueTimeout = 60 + +// The union's members, and the tag filter each owns. +var bazelDoltRaceMembers = []struct{ config, tag string }{ + {"doltserver", "dolt-server"}, + {"doltserver-proxied", "dolt-server-proxied"}, +} + +// bazelRCConfigLines returns .bazelrc's option lines for --config=name +// (any command: test:name, build:name, ...), as "