refactor(import): retire the same-chunk regular<->wisp deferral (wy-y52syc) (#6159)
* refactor(import): retire the same-chunk regular<->wisp deferral (wy-y52syc) wy-a648lq made the engine write an in-batch regular<->wisp edge under SkipDependencyValidationErrors (every row of both planes is on the one tx before PersistDependenciesWithOptionsResult), so the wy-4276q8 workaround in the classic import is redundant. Removed: splitDepsByChunk's same-chunk crossesBucket deferral, the wispByID map in partitionChunkedImportDeps, and importIssuesCore's hasCrossBucketBatchEdge reroute of a small import onto the chunked path. Also retired the dependency pass's single-plane split (splitDepRowsByBucket), which existed for the same filter and whose comment already named this bead. The plain later-chunk deferral is unchanged. Tests: the unit test for the same-chunk case now asserts the edge rides inline with no dependency pass; the small-batch test asserts one inline transaction; the embedded MixedRegularWisp test asserts one mixed pass; the resume-backfill drill is reshaped so a later-chunk cross-bucket related edge still defers and is backfilled on re-run, while the same-chunk blocks edge is durable after the crash. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JzJhoB3wAyxWDr5afcUskB * test(import): retire the dependency-pass narrative in the mixed in-batch case This PR inverted the routing for the two-row mixed regular/wisp fixture in TestImportFromLocalJSONL. At base the small-path guard also required `!hasCrossBucketBatchEdge(issues)`, so that batch took the chunked path and the regular -> wisp edge really was wired by the single-plane dependency pass (wy-4276q8). At HEAD the guard is `len(issues) <= importChunkSize` alone, so the batch takes the small path and the edge rides inline in its rows' own transaction. The assertions were already the correct post-change invariant, but the subtest name, the rationale comment, and the failure message still described the retired mechanism — the same narrative sweep this PR performed in the two test files it did touch. Prose only; assertions untouched. The comment now also records that this case is the real-engine guard for the small inline path, so its mock-based sibling (TestImportIssuesCoreSmallBatchCrossBucketEdgeStaysInline) is not mistaken for engine coverage. Maintainer review fixup for PR #6159. Refs wy-y52syc, wy-a648lq, wy-4276q8. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Eddie the Engineer <julianknutsen@users.noreply.github.com>
This commit is contained in:
parent
83184f4c6a
commit
e21eaf00ae
4 changed files with 137 additions and 245 deletions
|
|
@ -584,10 +584,11 @@ func TestImportChunkedStaleRejectedRowKeepsDeferredDepsOut(t *testing.T) {
|
|||
|
||||
// Cross-bucket (regular<->wisp) edges across a chunk boundary. Pre-wy-4276q8
|
||||
// the import applied the engine's per-batch cross-bucket filter over the FULL
|
||||
// set up front and so dropped every such edge; it now defers a same-chunk one
|
||||
// to a single-plane dependency pass (a workaround the engine no longer needs
|
||||
// since wy-a648lq, which writes in-batch cross-plane edges under
|
||||
// SkipDependencyValidationErrors; wy-y52syc retires it). This exercises the
|
||||
// set up front and so dropped every such edge; the engine now writes an
|
||||
// in-batch cross-plane edge under SkipDependencyValidationErrors (wy-a648lq),
|
||||
// so the import defers one only by the plain chunk rule (target first written
|
||||
// in a later chunk) and wires it in one mixed dependency pass (the
|
||||
// single-plane split of wy-4276q8 was retired by wy-y52syc). This exercises the
|
||||
// outcome end to end against the real engine: a regular -> wisp edge
|
||||
// straddling a chunk boundary must be wired, not skip-reported, and a
|
||||
// same-bucket wisp -> wisp edge straddling the same boundary must survive as
|
||||
|
|
@ -609,9 +610,9 @@ func TestImportChunkedMixedRegularWispCrossBucketEdgesWired(t *testing.T) {
|
|||
// 12 rows across three chunks (5/5/2) at chunk size 5. Every edge below is
|
||||
// `related` (non-readiness), so orderImportIssuesForChunking preserves file
|
||||
// order and the two forward edges genuinely straddle the chunk-0/chunk-1
|
||||
// boundary. Chunks 0 and 1 are each mixed (regular + wisp), so the engine's
|
||||
// per-chunk mixed-bucket filter actually runs instead of short-circuiting on
|
||||
// an all-one-bucket chunk.
|
||||
// boundary. Chunks 0 and 1 are each mixed (regular + wisp), so the rows of
|
||||
// both planes share one transaction — the shape wy-a648lq writes the
|
||||
// cross-plane edge under.
|
||||
makeIssues := func() []*types.Issue {
|
||||
issues := []*types.Issue{
|
||||
mk("bd-chunk01", false), // chunk 0: regular source of the cross-bucket edge
|
||||
|
|
@ -641,12 +642,11 @@ func TestImportChunkedMixedRegularWispCrossBucketEdgesWired(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("importIssuesCore: %v", err)
|
||||
}
|
||||
// 3 row chunks (5/5/2) plus TWO deferred-dependency passes: the regular
|
||||
// plane (bd-chunk01 -> bd-wisp07) and the wisp plane (bd-wisp02 ->
|
||||
// bd-wisp07) go in separate transactions, so the cross-bucket edge's
|
||||
// target is never in its batch.
|
||||
if counting.calls != 5 {
|
||||
t.Fatalf("calls = %d, want 3 row chunks + 2 single-plane deferred-dependency passes", counting.calls)
|
||||
// 3 row chunks (5/5/2) plus ONE mixed deferred-dependency pass carrying
|
||||
// both source rows (bd-chunk01 -> bd-wisp07 and bd-wisp02 -> bd-wisp07):
|
||||
// the pass no longer splits by plane.
|
||||
if counting.calls != 4 {
|
||||
t.Fatalf("calls = %d, want 3 row chunks + 1 mixed deferred-dependency pass", counting.calls)
|
||||
}
|
||||
if result.Created != 12 {
|
||||
t.Fatalf("Created = %d, want 12", result.Created)
|
||||
|
|
@ -686,13 +686,16 @@ func TestImportChunkedMixedRegularWispCrossBucketEdgesWired(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// The wy-4276q8 rollback-drill shape against the real engine: a regular issue
|
||||
// blocked by a wisp created in the same chunk (deferred edge) and a wisp
|
||||
// blocked by a regular issue from an earlier chunk (inline edge). The import
|
||||
// dies after the row chunks and before the dependency pass — where every drill
|
||||
// attempt died — so the inline edge is durable and the deferred one is not yet;
|
||||
// re-running the same import must backfill it with nothing skip-reported, and
|
||||
// a further run must be a no-op for the edges.
|
||||
// The wy-4276q8 rollback-drill shape against the real engine, after the
|
||||
// same-chunk deferral's retirement (wy-y52syc): a wisp blocked by a regular
|
||||
// issue created in the SAME chunk (an inline edge now), a wisp blocked by a
|
||||
// regular issue from an earlier chunk (inline edge), and a regular issue
|
||||
// related to a wisp first written in a LATER chunk (still deferred, by the
|
||||
// plain chunk rule). The import dies after the row chunks and before the
|
||||
// dependency pass — where every drill attempt died — so both inline edges are
|
||||
// durable and the deferred one is not yet; re-running the same import must
|
||||
// backfill it with nothing skip-reported, and a further run must be a no-op
|
||||
// for the edges.
|
||||
func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) {
|
||||
requireEmbeddedDolt(t)
|
||||
setImportChunkSize(t, 4)
|
||||
|
|
@ -701,21 +704,30 @@ func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) {
|
|||
ctx := context.Background()
|
||||
store := provisionChunkStore(t)
|
||||
|
||||
// Kahn's ordering yields chunk 0: f1 f2 f3 f4 chunk 1: w1 w2 r1
|
||||
// r1 -> w1 is same-chunk cross-bucket (deferred); w2 -> f1 points at an
|
||||
// earlier chunk (inline).
|
||||
// Kahn's ordering (indegree-0 rows in file order, dependents released
|
||||
// behind them) yields
|
||||
// chunk 0: f1 f2 f3 f4 chunk 1: r1 f5 f6 f7 chunk 2: r2 w2 w1
|
||||
// w1 -> r2 is same-chunk cross-bucket (inline since wy-y52syc); w2 -> f1
|
||||
// points at an earlier chunk (inline); r1 -related-> w2 points at a LATER
|
||||
// chunk (deferred).
|
||||
makeIssues := func() []*types.Issue {
|
||||
issues := []*types.Issue{
|
||||
crossBucketTestIssue("bd-f1", false),
|
||||
crossBucketTestIssue("bd-f2", false),
|
||||
crossBucketTestIssue("bd-f3", false),
|
||||
crossBucketTestIssue("bd-f4", false),
|
||||
crossBucketTestIssue("bd-w1", true),
|
||||
crossBucketTestIssue("bd-r1", false),
|
||||
crossBucketTestIssue("bd-f5", false),
|
||||
crossBucketTestIssue("bd-f6", false),
|
||||
crossBucketTestIssue("bd-f7", false),
|
||||
crossBucketTestIssue("bd-w1", true),
|
||||
crossBucketTestIssue("bd-r2", false),
|
||||
crossBucketTestIssue("bd-w2", true),
|
||||
}
|
||||
issues[5].Dependencies = []*types.Dependency{{IssueID: "bd-r1", DependsOnID: "bd-w1", Type: types.DepBlocks}}
|
||||
issues[6].Dependencies = []*types.Dependency{{IssueID: "bd-w2", DependsOnID: "bd-f1", Type: types.DepBlocks}}
|
||||
issues[4].Dependencies = []*types.Dependency{{IssueID: "bd-r1", DependsOnID: "bd-w2", Type: types.DepRelated}}
|
||||
issues[8].Dependencies = []*types.Dependency{{IssueID: "bd-w1", DependsOnID: "bd-r2", Type: types.DepBlocks}}
|
||||
issues[9].Dependencies = []*types.Dependency{{IssueID: "bd-r2", DependsOnID: "bd-f1", Type: types.DepBlocks}}
|
||||
issues[10].Dependencies = []*types.Dependency{{IssueID: "bd-w2", DependsOnID: "bd-f1", Type: types.DepBlocks}}
|
||||
return issues
|
||||
}
|
||||
depIDs := func(id string) []string {
|
||||
|
|
@ -731,23 +743,26 @@ func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) {
|
|||
return ids
|
||||
}
|
||||
|
||||
failing := &failNthCreateStore{DoltStorage: store, failOnCall: 3}
|
||||
failing := &failNthCreateStore{DoltStorage: store, failOnCall: 4}
|
||||
if _, err := importIssuesCore(ctx, "", failing, makeIssues(), ImportOptions{SkipPrefixValidation: true}); err == nil {
|
||||
t.Fatalf("importIssuesCore succeeded, want the simulated crash in the dependency pass")
|
||||
}
|
||||
if failing.calls != 3 {
|
||||
t.Fatalf("calls = %d, want 2 row chunks + the failing dependency pass", failing.calls)
|
||||
if failing.calls != 4 {
|
||||
t.Fatalf("calls = %d, want 3 row chunks + the failing dependency pass", failing.calls)
|
||||
}
|
||||
for _, id := range []string{"bd-f1", "bd-f2", "bd-f3", "bd-f4", "bd-w1", "bd-r1", "bd-w2"} {
|
||||
for _, id := range []string{"bd-f1", "bd-f2", "bd-f3", "bd-f4", "bd-r1", "bd-f5", "bd-f6", "bd-f7", "bd-w1", "bd-r2", "bd-w2"} {
|
||||
if _, err := store.GetIssue(ctx, id); err != nil {
|
||||
t.Fatalf("row %s not durable after the crash: %v", id, err)
|
||||
}
|
||||
}
|
||||
if got := depIDs("bd-w1"); len(got) != 1 || got[0] != "bd-r2" {
|
||||
t.Fatalf("bd-w1 deps after crash = %v, want the same-chunk cross-bucket edge [bd-r2] already durable (inline since wy-y52syc)", got)
|
||||
}
|
||||
if got := depIDs("bd-w2"); len(got) != 1 || got[0] != "bd-f1" {
|
||||
t.Fatalf("bd-w2 deps after crash = %v, want the inline cross-bucket edge [bd-f1] already durable", got)
|
||||
}
|
||||
if got := depIDs("bd-r1"); len(got) != 0 {
|
||||
t.Fatalf("bd-r1 deps after crash = %v, want none yet (deferred edge lost with the dependency pass)", got)
|
||||
t.Fatalf("bd-r1 deps after crash = %v, want none yet (deferred later-chunk edge lost with the dependency pass)", got)
|
||||
}
|
||||
|
||||
// The re-run backfills the deferred edge: this is the drill's "re-run
|
||||
|
|
@ -759,8 +774,11 @@ func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) {
|
|||
if len(result.SkippedDependencies) != 0 {
|
||||
t.Fatalf("re-run SkippedDependencies = %#v, want none", result.SkippedDependencies)
|
||||
}
|
||||
if got := depIDs("bd-r1"); len(got) != 1 || got[0] != "bd-w1" {
|
||||
t.Fatalf("bd-r1 deps after re-run = %v, want the cross-bucket edge [bd-w1] backfilled", got)
|
||||
if got := depIDs("bd-r1"); len(got) != 1 || got[0] != "bd-w2" {
|
||||
t.Fatalf("bd-r1 deps after re-run = %v, want the later-chunk cross-bucket edge [bd-w2] backfilled", got)
|
||||
}
|
||||
if got := depIDs("bd-w1"); len(got) != 1 || got[0] != "bd-r2" {
|
||||
t.Fatalf("bd-w1 deps after re-run = %v, want [bd-r2] kept", got)
|
||||
}
|
||||
if got := depIDs("bd-w2"); len(got) != 1 || got[0] != "bd-f1" {
|
||||
t.Fatalf("bd-w2 deps after re-run = %v, want [bd-f1] kept", got)
|
||||
|
|
@ -774,7 +792,7 @@ func TestImportChunkedCrossBucketBlocksEdgesResumeBackfills(t *testing.T) {
|
|||
if len(result.SkippedDependencies) != 0 {
|
||||
t.Fatalf("third run SkippedDependencies = %#v, want none", result.SkippedDependencies)
|
||||
}
|
||||
if got := depIDs("bd-r1"); len(got) != 1 || got[0] != "bd-w1" {
|
||||
t.Fatalf("bd-r1 deps after third run = %v, want [bd-w1]", got)
|
||||
if got := depIDs("bd-r1"); len(got) != 1 || got[0] != "bd-w2" {
|
||||
t.Fatalf("bd-r1 deps after third run = %v, want [bd-w2]", got)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -347,28 +347,21 @@ func crossBucketTestIssue(id string, wisp bool) *types.Issue {
|
|||
return issue
|
||||
}
|
||||
|
||||
// assertNoSameBatchCrossBucketEdge fails if any recorded batch carries an edge
|
||||
// whose target is a row of that same batch on the other plane — the exact
|
||||
// shape the engine's per-batch cross-bucket filter skip-reports as a
|
||||
// cross-bucket dependency. It also fails if an edge commits
|
||||
// before its target row exists (target first written in a later batch).
|
||||
func assertNoSameBatchCrossBucketEdge(t *testing.T, store *chunkRecordingStore) {
|
||||
// assertEdgesCommitAfterTargets fails if any recorded batch carries an edge
|
||||
// that commits before its target row exists (target first written in a later
|
||||
// batch). A regular<->wisp edge whose target is a row of the same batch is
|
||||
// fine: the engine writes it (wy-a648lq), so the import no longer defers it.
|
||||
func assertEdgesCommitAfterTargets(t *testing.T, store *chunkRecordingStore) {
|
||||
t.Helper()
|
||||
firstBatchOf := map[string]int{}
|
||||
wispByID := map[string]bool{}
|
||||
for b, batch := range store.batches {
|
||||
for _, issue := range batch {
|
||||
if _, ok := firstBatchOf[issue.ID]; !ok {
|
||||
firstBatchOf[issue.ID] = b
|
||||
}
|
||||
wispByID[issue.ID] = issue.Ephemeral
|
||||
}
|
||||
}
|
||||
for b, batch := range store.batches {
|
||||
inBatch := map[string]bool{}
|
||||
for _, issue := range batch {
|
||||
inBatch[issue.ID] = true
|
||||
}
|
||||
for _, issue := range batch {
|
||||
for _, dep := range issue.Dependencies {
|
||||
tb, ok := firstBatchOf[dep.DependsOnID]
|
||||
|
|
@ -378,9 +371,6 @@ func assertNoSameBatchCrossBucketEdge(t *testing.T, store *chunkRecordingStore)
|
|||
if tb > b {
|
||||
t.Fatalf("batch %d: %s -> %s commits before its target exists (target first written in batch %d)", b, issue.ID, dep.DependsOnID, tb)
|
||||
}
|
||||
if inBatch[dep.DependsOnID] && wispByID[issue.ID] != wispByID[dep.DependsOnID] {
|
||||
t.Fatalf("batch %d: %s -> %s is a regular<->wisp edge whose target is in the same batch; the engine would skip-report it as cross-bucket", b, issue.ID, dep.DependsOnID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -400,25 +390,23 @@ func depTargetsIn(store *chunkRecordingStore, b int, id string) []string {
|
|||
return targets
|
||||
}
|
||||
|
||||
// A regular<->wisp edge whose endpoints are created in the same chunk cannot be
|
||||
// wired in that chunk's transaction (the engine's per-batch cross-bucket
|
||||
// filter skip-reports it), so the
|
||||
// import must defer it to the dependency pass, while one into an earlier chunk
|
||||
// points at a committed row and rides inline. Every dependency-pass
|
||||
// transaction must be single-plane, or a deferred edge's target could sit in
|
||||
// the same batch on the other plane and be skipped all over again. Pre-fix the
|
||||
// import filtered every in-batch cross-bucket edge out up front and reported
|
||||
// it skipped — export→import lost every such edge, and a re-run could never
|
||||
// backfill it (wy-4276q8).
|
||||
func TestImportChunkedCrossBucketEdgesDeferredToSinglePlanePass(t *testing.T) {
|
||||
// A regular<->wisp edge whose endpoints are created in the same chunk rides
|
||||
// inline with its row, exactly like a same-plane edge: the engine writes an
|
||||
// in-batch cross-plane edge under SkipDependencyValidationErrors (wy-a648lq).
|
||||
// The import once deferred such an edge to a single-plane dependency pass to
|
||||
// dodge the engine's per-batch cross-bucket filter (wy-4276q8, which had lost
|
||||
// every such edge on export→import); wy-y52syc retired that deferral, so a
|
||||
// batch whose edges all point at same- or earlier-chunk rows needs no
|
||||
// dependency pass at all.
|
||||
func TestImportChunkedSameChunkCrossBucketEdgesRideInline(t *testing.T) {
|
||||
setImportChunkSize(t, 4)
|
||||
recordImportPauses(t)
|
||||
setImportProgressBuffer(t)
|
||||
// File order and blocking edges chosen so that Kahn's ordering (indegree-0
|
||||
// rows in file order, dependents released behind them) yields
|
||||
// chunk 0: f1 f2 f3 f4 chunk 1: w1 r2 w3 r1 chunk 2: w2
|
||||
// r1 -> w1 lands in the same chunk as its wisp target (must defer);
|
||||
// w3 -> f1 and w2 -> r2 point at earlier chunks (must ride inline).
|
||||
// r1 -> w1 lands in the same chunk as its wisp target (rides inline);
|
||||
// w3 -> f1 and w2 -> r2 point at earlier chunks (ride inline too).
|
||||
issues := []*types.Issue{
|
||||
crossBucketTestIssue("bd-f1", false),
|
||||
crossBucketTestIssue("bd-f2", false),
|
||||
|
|
@ -439,29 +427,19 @@ func TestImportChunkedCrossBucketEdgesDeferredToSinglePlanePass(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("importIssuesCore: %v", err)
|
||||
}
|
||||
if store.calls != 4 {
|
||||
t.Fatalf("calls = %d, want 3 row chunks + 1 single-plane dependency pass", store.calls)
|
||||
if store.calls != 3 {
|
||||
t.Fatalf("calls = %d, want 3 row chunks and NO dependency pass", store.calls)
|
||||
}
|
||||
for b := 0; b < 3; b++ {
|
||||
if store.conflictSkip[b] {
|
||||
t.Fatalf("batch %d is a row chunk but was submitted with ConflictSkip", b)
|
||||
}
|
||||
}
|
||||
if !store.conflictSkip[3] {
|
||||
t.Fatalf("batch 3 must be the dependency pass (ConflictSkip)")
|
||||
}
|
||||
assertNoSameBatchCrossBucketEdge(t, store)
|
||||
assertEdgesCommitAfterTargets(t, store)
|
||||
|
||||
// The same-chunk cross-bucket edge is stripped from its row write and wired
|
||||
// by the pass; the pass carries only that row.
|
||||
if got := depTargetsIn(store, 1, "bd-r1"); len(got) != 0 {
|
||||
t.Fatalf("bd-r1 row chunk carries deps %v, want none (same-chunk wisp target must be deferred)", got)
|
||||
}
|
||||
if len(store.batches[3]) != 1 || store.batches[3][0].ID != "bd-r1" {
|
||||
t.Fatalf("dependency pass rows = %v, want exactly bd-r1", store.batches[3])
|
||||
}
|
||||
if got := depTargetsIn(store, 3, "bd-r1"); len(got) != 1 || got[0] != "bd-w1" {
|
||||
t.Fatalf("dependency pass deps for bd-r1 = %v, want [bd-w1]", got)
|
||||
// The same-chunk cross-bucket edge rides inline with its row.
|
||||
if got := depTargetsIn(store, 1, "bd-r1"); len(got) != 1 || got[0] != "bd-w1" {
|
||||
t.Fatalf("bd-r1 inline deps = %v, want [bd-w1] (same-chunk wisp target rides inline)", got)
|
||||
}
|
||||
// Cross-bucket edges into earlier chunks ride inline with their rows.
|
||||
if got := depTargetsIn(store, 1, "bd-w3"); len(got) != 1 || got[0] != "bd-f1" {
|
||||
|
|
@ -485,11 +463,11 @@ func TestImportChunkedCrossBucketEdgesDeferredToSinglePlanePass(t *testing.T) {
|
|||
}
|
||||
|
||||
// A small import (at or below the chunk size) keeps its single transaction
|
||||
// unless it carries a regular<->wisp edge between two of its own rows: that
|
||||
// edge would be skip-reported by the engine's per-batch cross-bucket filter,
|
||||
// so the import takes the chunked path and the edge lands in a dependency pass
|
||||
// instead of being skip-reported.
|
||||
func TestImportIssuesCoreSmallBatchCrossBucketEdgeTakesDependencyPass(t *testing.T) {
|
||||
// even when it carries a regular<->wisp edge between two of its own rows: the
|
||||
// engine writes that edge under SkipDependencyValidationErrors (wy-a648lq),
|
||||
// so the chunked-path detour that once landed it in a dependency pass
|
||||
// (wy-4276q8) is retired (wy-y52syc).
|
||||
func TestImportIssuesCoreSmallBatchCrossBucketEdgeStaysInline(t *testing.T) {
|
||||
setImportChunkSize(t, 4)
|
||||
recordImportPauses(t)
|
||||
setImportProgressBuffer(t)
|
||||
|
|
@ -504,21 +482,15 @@ func TestImportIssuesCoreSmallBatchCrossBucketEdgeTakesDependencyPass(t *testing
|
|||
if err != nil {
|
||||
t.Fatalf("importIssuesCore: %v", err)
|
||||
}
|
||||
if store.calls != 2 {
|
||||
t.Fatalf("calls = %d, want 1 row chunk + 1 dependency pass for a small import with a cross-bucket edge", store.calls)
|
||||
}
|
||||
if store.conflictSkip[0] || !store.conflictSkip[1] {
|
||||
t.Fatalf("conflictSkip = %v, want [false true] (row chunk, then dependency pass)", store.conflictSkip)
|
||||
if store.calls != 1 || store.conflictSkip[0] {
|
||||
t.Fatalf("calls = %d conflictSkip = %v, want one inline transaction for a small import with a cross-bucket edge", store.calls, store.conflictSkip)
|
||||
}
|
||||
if len(store.batches[0]) != 2 {
|
||||
t.Fatalf("row chunk size = %d, want both rows", len(store.batches[0]))
|
||||
}
|
||||
assertNoSameBatchCrossBucketEdge(t, store)
|
||||
if got := depTargetsIn(store, 0, "bd-r1"); len(got) != 0 {
|
||||
t.Fatalf("row chunk carries %v for bd-r1, want none", got)
|
||||
}
|
||||
if got := depTargetsIn(store, 1, "bd-r1"); len(got) != 1 || got[0] != "bd-w1" {
|
||||
t.Fatalf("dependency pass deps for bd-r1 = %v, want [bd-w1]", got)
|
||||
assertEdgesCommitAfterTargets(t, store)
|
||||
if got := depTargetsIn(store, 0, "bd-r1"); len(got) != 1 || got[0] != "bd-w1" {
|
||||
t.Fatalf("inline deps for bd-r1 = %v, want [bd-w1]", got)
|
||||
}
|
||||
if result.Created != 2 {
|
||||
t.Fatalf("Created = %d, want 2", result.Created)
|
||||
|
|
|
|||
|
|
@ -400,7 +400,7 @@ func TestImportFromLocalJSONL(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
t.Run("wires mixed regular and wisp in-batch dependencies through the dependency pass", func(t *testing.T) {
|
||||
t.Run("wires mixed regular and wisp in-batch dependencies inline", func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
dbPath := filepath.Join(tmpDir, "dolt")
|
||||
store := newTestStore(t, dbPath)
|
||||
|
|
@ -440,9 +440,15 @@ func TestImportFromLocalJSONL(t *testing.T) {
|
|||
if result.Created != 2 {
|
||||
t.Fatalf("Created = %d, want 2", result.Created)
|
||||
}
|
||||
// The engine's per-batch cross-bucket filter would skip-report the
|
||||
// regular -> wisp edge in a mixed batch, so the import defers it to a
|
||||
// single-plane dependency pass instead (wy-4276q8).
|
||||
// The engine writes an in-batch cross-plane edge under
|
||||
// SkipDependencyValidationErrors (wy-a648lq), so this two-row mixed
|
||||
// batch takes the small path and the regular -> wisp edge rides inline
|
||||
// in its rows' own transaction; the chunked-path detour through a
|
||||
// single-plane dependency pass it once needed (wy-4276q8) was retired
|
||||
// by wy-y52syc. This is the real-engine guard for the small inline
|
||||
// path's cross-plane behavior: the unit test's recording store
|
||||
// (TestImportIssuesCoreSmallBatchCrossBucketEdgeStaysInline) pins the
|
||||
// routing shape, this case pins the outcome against a real store.
|
||||
if len(result.SkippedDependencies) != 0 {
|
||||
t.Fatalf("SkippedDependencies = %#v, want none", result.SkippedDependencies)
|
||||
}
|
||||
|
|
@ -457,7 +463,7 @@ func TestImportFromLocalJSONL(t *testing.T) {
|
|||
t.Fatalf("GetDependencyRecords(test-mixed-regular): %v", err)
|
||||
}
|
||||
if len(deps) != 1 || deps[0].DependsOnID != "test-mixed-wisp" {
|
||||
t.Fatalf("test-mixed-regular deps = %#v, want the regular -> wisp edge wired by the dependency pass", deps)
|
||||
t.Fatalf("test-mixed-regular deps = %#v, want the regular -> wisp edge wired inline", deps)
|
||||
}
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -188,14 +188,12 @@ func importIssuesCore(ctx context.Context, _ string, store storage.DoltStorage,
|
|||
},
|
||||
}
|
||||
var err error
|
||||
if len(issues) <= importChunkSize && !hasCrossBucketBatchEdge(issues) {
|
||||
if len(issues) <= importChunkSize {
|
||||
// Small import: one transaction, dependencies inline — exactly the
|
||||
// pre-chunking behavior. A batch carrying a regular<->wisp edge still
|
||||
// takes the chunked path: until wy-a648lq the engine's per-batch
|
||||
// cross-bucket filter skip-reported such an edge whenever both
|
||||
// endpoints were rows of one mixed batch, so it needed the single-plane
|
||||
// dependency pass; the engine now writes it inline, and wy-y52syc
|
||||
// retires this routing.
|
||||
// pre-chunking behavior. A regular<->wisp edge between two of its
|
||||
// rows rides inline too: the engine writes an in-batch cross-plane
|
||||
// edge under SkipDependencyValidationErrors (wy-a648lq), so the
|
||||
// chunked-path detour it once needed (wy-4276q8) is gone (wy-y52syc).
|
||||
err = store.CreateIssuesWithFullOptions(ctx, issues, actor, batchOpts)
|
||||
} else {
|
||||
err = importIssuesChunked(ctx, store, issues, actor, batchOpts)
|
||||
|
|
@ -258,31 +256,22 @@ func assembleImportResult(issues []*types.Issue, staleSkippedIDs []string, chang
|
|||
// or an earlier chunk — Kahn's order for the acyclic majority, plus a
|
||||
// cycle-breaking fallback that still emits each cycle member before the rows it
|
||||
// blocks — so that edge rides inline with its row and both commit in one
|
||||
// transaction. Only a force-emitted cycle member — or a row whose blocker is a
|
||||
// same-chunk row on the other plane (regular<->wisp, see below) — can carry a
|
||||
// readiness edge into the deferred pass. Outside those two cases a concurrent
|
||||
// reader never observes an imported bead without the blocking edges its
|
||||
// import file declares; for the cross-bucket case the window lasts until the
|
||||
// dependency pass, and a crash before it leaves the bead spuriously ready until
|
||||
// the import is re-run (which backfills the edge).
|
||||
// transaction. Only a force-emitted cycle member can carry a readiness edge
|
||||
// into the deferred pass; outside that case a concurrent reader never observes
|
||||
// an imported bead without the blocking edges its import file declares.
|
||||
//
|
||||
// Only edges that cannot be satisfied when their row commits are deferred to
|
||||
// a final dependency pass: edges into an intra-batch dependency cycle
|
||||
// (invalid for blocking types; still broken and skip-reported at wire time,
|
||||
// though for a cycle of length >=3 the specific edge dropped — and thus which
|
||||
// member is left spuriously ready — can differ from the single-transaction
|
||||
// import, which checks the whole cycle in file order), non-readiness edges
|
||||
// (related, discovered-from) that point at a later chunk, and regular<->wisp
|
||||
// edges whose target is first written in the same chunk: until wy-a648lq the
|
||||
// engine's per-batch cross-bucket filter skip-reported any regular<->wisp edge
|
||||
// whose endpoints were both rows of one mixed batch, whatever the transaction
|
||||
// layout (it now writes such an edge under SkipDependencyValidationErrors, and
|
||||
// wy-y52syc retires this deferral). Deferring such an edge opens
|
||||
// a ready window like a cycle member's deferred edge — and a common one, since
|
||||
// Kahn's order tends to place a blocker directly before its dependent, i.e. in
|
||||
// the same chunk; dropping it — the import's former policy — lost it for good,
|
||||
// and a re-run could never backfill it (wy-4276q8: every regular<->wisp
|
||||
// `blocks` edge of a wyvern export was lost on restore). The dependency pass submits
|
||||
// import, which checks the whole cycle in file order) and non-readiness edges
|
||||
// (related, discovered-from) that point at a later chunk. A regular<->wisp
|
||||
// edge is deferred by those rules alone, never for crossing planes: the
|
||||
// engine writes an in-batch cross-plane edge under
|
||||
// SkipDependencyValidationErrors (wy-a648lq), so the same-chunk deferral the
|
||||
// import once needed (wy-4276q8: every regular<->wisp `blocks` edge of a
|
||||
// wyvern export was lost on restore) is retired (wy-y52syc). The dependency pass submits
|
||||
// row copies stripped to those deferred edges with ConflictSkip set, so an
|
||||
// existing row is never rewritten: a concurrent update landing between a
|
||||
// row's chunk and the dependency pass cannot stale-reject the pass and drop
|
||||
|
|
@ -301,18 +290,15 @@ func assembleImportResult(issues []*types.Issue, staleSkippedIDs []string, chang
|
|||
// still-unwired deferred edges), reported in StaleSkippedIDs. That is the
|
||||
// same local-wins outcome the single-transaction import gave a rival update
|
||||
// racing a crashed import, and it can only affect deferred edges (non-readiness
|
||||
// edges, the readiness edges of force-emitted cycle members, and same-chunk
|
||||
// regular<->wisp readiness edges); every other readiness edge commits with its
|
||||
// row.
|
||||
// edges and the readiness edges of force-emitted cycle members); every other
|
||||
// readiness edge commits with its row.
|
||||
func importIssuesChunked(ctx context.Context, store storage.DoltStorage, issues []*types.Issue, actor string, opts storage.BatchCreateOptions) error {
|
||||
// Cross-bucket (regular<->wisp) edges are never filtered: since wy-a648lq
|
||||
// the engine writes an in-batch one under SkipDependencyValidationErrors
|
||||
// (every row of both planes is on the chunk's one tx before its dependency
|
||||
// pass). partitionChunkedImportDeps still defers one whose target is first
|
||||
// written in the SAME chunk to the single-plane dependency pass — the
|
||||
// wy-4276q8 workaround for the per-batch filter the engine no longer
|
||||
// applies, retired by wy-y52syc — and wires one into an earlier chunk
|
||||
// inline (its target is then a committed row). See the ordering notes above.
|
||||
// pass), so a same-chunk one rides inline like any other edge and only the
|
||||
// chunk rule (target first written in a later chunk) defers. See the
|
||||
// ordering notes above.
|
||||
ordered := orderImportIssuesForChunking(issues)
|
||||
|
||||
// Partitioning narrows each issue's dependency slice to its inline subset in
|
||||
|
|
@ -378,13 +364,9 @@ type deferredImportEdges struct {
|
|||
// chunk, so only non-readiness edges (related, discovered-from) into a later
|
||||
// chunk and the readiness edges of force-emitted cycle members (their own cyclic
|
||||
// edges, plus any acyclic blocker they were emitted ahead of) are ever deferred
|
||||
// here — plus regular<->wisp edges whose target is first written in the same
|
||||
// chunk (see splitDepsByChunk).
|
||||
// here.
|
||||
func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges {
|
||||
firstChunkOf := make(map[string]int, len(ordered))
|
||||
// Last row wins for the bucket, mirroring the engine's own per-batch
|
||||
// plane check (issueops.validateCreateIssuesMixedBucketDependencies).
|
||||
wispByID := make(map[string]bool, len(ordered))
|
||||
for pos, issue := range ordered {
|
||||
if issue.ID == "" {
|
||||
continue // ID assigned by the engine at insert; nothing can reference it yet
|
||||
|
|
@ -392,14 +374,13 @@ func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges {
|
|||
if _, ok := firstChunkOf[issue.ID]; !ok {
|
||||
firstChunkOf[issue.ID] = pos / importChunkSize
|
||||
}
|
||||
wispByID[issue.ID] = issueops.IsWisp(issue)
|
||||
}
|
||||
var deferred []deferredImportEdges
|
||||
for pos, issue := range ordered {
|
||||
if len(issue.Dependencies) == 0 {
|
||||
continue
|
||||
}
|
||||
inline, later := splitDepsByChunk(issue.Dependencies, pos/importChunkSize, issueops.IsWisp(issue), firstChunkOf, wispByID)
|
||||
inline, later := splitDepsByChunk(issue.Dependencies, pos/importChunkSize, firstChunkOf)
|
||||
if len(later) == 0 {
|
||||
continue
|
||||
}
|
||||
|
|
@ -411,26 +392,18 @@ func partitionChunkedImportDeps(ordered []*types.Issue) []deferredImportEdges {
|
|||
|
||||
// splitDepsByChunk partitions one row's dependencies (the row lands in rowChunk)
|
||||
// into edges that can be wired inline and edges that must be deferred: the
|
||||
// target is first written in a later chunk, or the edge crosses the
|
||||
// regular/wisp bucket boundary and its target is first written in the SAME
|
||||
// chunk. Until wy-a648lq the engine's per-batch cross-bucket filter
|
||||
// skip-reported a regular<->wisp edge whose endpoints were both rows of one
|
||||
// mixed batch, and the import lost it (wy-4276q8); the engine now writes such
|
||||
// an edge, and this deferral is kept only until wy-y52syc retires it. A
|
||||
// cross-bucket edge into an earlier chunk points at a committed row outside
|
||||
// the batch and rides inline like any other.
|
||||
func splitDepsByChunk(deps []*types.Dependency, rowChunk int, rowIsWisp bool, firstChunkOf map[string]int, wispByID map[string]bool) (inline, later []*types.Dependency) {
|
||||
// target is first written in a later chunk. Plane is not a criterion — a
|
||||
// regular<->wisp edge whose target is a same- or earlier-chunk row rides
|
||||
// inline, since the engine writes an in-batch cross-plane edge under
|
||||
// SkipDependencyValidationErrors (wy-a648lq; the same-chunk deferral of
|
||||
// wy-4276q8 was retired by wy-y52syc).
|
||||
func splitDepsByChunk(deps []*types.Dependency, rowChunk int, firstChunkOf map[string]int) (inline, later []*types.Dependency) {
|
||||
for _, dep := range deps {
|
||||
if dep == nil {
|
||||
inline = append(inline, dep)
|
||||
continue
|
||||
}
|
||||
targetChunk, inBatch := firstChunkOf[dep.DependsOnID]
|
||||
if inBatch && targetChunk > rowChunk {
|
||||
later = append(later, dep)
|
||||
continue
|
||||
}
|
||||
if inBatch && targetChunk == rowChunk && crossesBucket(dep, rowIsWisp, wispByID) {
|
||||
if targetChunk, inBatch := firstChunkOf[dep.DependsOnID]; inBatch && targetChunk > rowChunk {
|
||||
later = append(later, dep)
|
||||
continue
|
||||
}
|
||||
|
|
@ -439,49 +412,6 @@ func splitDepsByChunk(deps []*types.Dependency, rowChunk int, rowIsWisp bool, fi
|
|||
return inline, later
|
||||
}
|
||||
|
||||
// crossesBucket reports whether dep joins a regular issue and a wisp. The
|
||||
// source bucket is the owning row's unless the edge names another batch row
|
||||
// as its source, mirroring the engine's per-batch check; the target must be a
|
||||
// batch row (wispByID is keyed by batch IDs), so callers check membership.
|
||||
func crossesBucket(dep *types.Dependency, rowIsWisp bool, wispByID map[string]bool) bool {
|
||||
sourceIsWisp := rowIsWisp
|
||||
if dep.IssueID != "" {
|
||||
if isWisp, ok := wispByID[dep.IssueID]; ok {
|
||||
sourceIsWisp = isWisp
|
||||
}
|
||||
}
|
||||
return sourceIsWisp != wispByID[dep.DependsOnID]
|
||||
}
|
||||
|
||||
// hasCrossBucketBatchEdge reports whether any row's dependency points at
|
||||
// another row of the same batch on the other plane (regular<->wisp). Until
|
||||
// wy-a648lq the engine's per-batch cross-bucket filter skip-reported such an
|
||||
// edge, so an import carrying one takes the chunked path and defers it to the
|
||||
// single-plane dependency pass; the engine now writes it inline, and
|
||||
// wy-y52syc retires this routing.
|
||||
func hasCrossBucketBatchEdge(issues []*types.Issue) bool {
|
||||
wispByID := make(map[string]bool, len(issues))
|
||||
for _, issue := range issues {
|
||||
if issue != nil && issue.ID != "" {
|
||||
wispByID[issue.ID] = issueops.IsWisp(issue)
|
||||
}
|
||||
}
|
||||
for _, issue := range issues {
|
||||
if issue == nil {
|
||||
continue
|
||||
}
|
||||
for _, dep := range issue.Dependencies {
|
||||
if dep == nil {
|
||||
continue
|
||||
}
|
||||
if _, inBatch := wispByID[dep.DependsOnID]; inBatch && crossesBucket(dep, issueops.IsWisp(issue), wispByID) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// writeImportRowChunks writes the ordered rows (dependencies already narrowed to
|
||||
// their inline subset) in bounded transactions, pausing between commits.
|
||||
func writeImportRowChunks(ctx context.Context, store storage.DoltStorage, ordered []*types.Issue, actor string, rowOpts storage.BatchCreateOptions, pacer *importChunkPacer) error {
|
||||
|
|
@ -528,58 +458,24 @@ func wireDeferredImportDeps(ctx context.Context, store storage.DoltStorage, defe
|
|||
// callback unset keeps a phase-2 signal from ever misreporting a row
|
||||
// whose phase-1 write committed.
|
||||
depOpts.OnStaleRejected = nil
|
||||
// Regular-source rows and wisp-source rows go in separate transactions.
|
||||
// Until wy-a648lq the engine's per-batch cross-bucket filter skip-reported
|
||||
// a regular<->wisp edge whose target row was in the same mixed batch, and
|
||||
// a deferred cross-bucket edge's target may itself carry deferred edges —
|
||||
// a mixed batch could re-trigger exactly the skip this pass existed to
|
||||
// avoid. (The engine now writes such an edge; wy-y52syc retires the split.)
|
||||
// That filter short-circuited on a single-plane batch, so with every batch
|
||||
// on one plane nothing can be filtered, and every target is a committed
|
||||
// phase-1 row.
|
||||
// One plane per transaction is no longer required: the engine writes an
|
||||
// in-batch regular<->wisp edge under SkipDependencyValidationErrors
|
||||
// (wy-a648lq), so a mixed dependency-pass batch wires every edge whose
|
||||
// target is a committed phase-1 row (the single-plane split of wy-4276q8
|
||||
// was retired by wy-y52syc).
|
||||
depTotal := len(depRows)
|
||||
planes := splitDepRowsByBucket(depRows)
|
||||
depChunks := 0
|
||||
for _, plane := range planes {
|
||||
depChunks += (len(plane) + importChunkSize - 1) / importChunkSize
|
||||
}
|
||||
wired, chunk := 0, 1
|
||||
for _, plane := range planes {
|
||||
for start := 0; start < len(plane); start += importChunkSize {
|
||||
end := min(start+importChunkSize, len(plane))
|
||||
pacer.beforeTx()
|
||||
if err := store.CreateIssuesWithFullOptions(ctx, plane[start:end], actor, depOpts); err != nil {
|
||||
return fmt.Errorf("import dependency pass chunk %d/%d failed (all %d issue rows are committed; re-run the import to resume — it converges): %w", chunk, depChunks, rowTotal, err)
|
||||
}
|
||||
wired += end - start
|
||||
chunk++
|
||||
fmt.Fprintf(importProgress, "bd import: deferred dependencies wired for %d/%d issues\n", wired, depTotal) //nolint:gosec // G705: stderr, not a browser context
|
||||
depChunks := (depTotal + importChunkSize - 1) / importChunkSize
|
||||
for start := 0; start < depTotal; start += importChunkSize {
|
||||
end := min(start+importChunkSize, depTotal)
|
||||
pacer.beforeTx()
|
||||
if err := store.CreateIssuesWithFullOptions(ctx, depRows[start:end], actor, depOpts); err != nil {
|
||||
return fmt.Errorf("import dependency pass chunk %d/%d failed (all %d issue rows are committed; re-run the import to resume — it converges): %w", start/importChunkSize+1, depChunks, rowTotal, err)
|
||||
}
|
||||
fmt.Fprintf(importProgress, "bd import: deferred dependencies wired for %d/%d issues\n", end, depTotal) //nolint:gosec // G705: stderr, not a browser context
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitDepRowsByBucket partitions dependency-pass rows into the regular plane
|
||||
// and the wisp plane, each in its original order; an empty plane is omitted.
|
||||
func splitDepRowsByBucket(rows []*types.Issue) [][]*types.Issue {
|
||||
var regular, wisp []*types.Issue
|
||||
for _, row := range rows {
|
||||
if issueops.IsWisp(row) {
|
||||
wisp = append(wisp, row)
|
||||
} else {
|
||||
regular = append(regular, row)
|
||||
}
|
||||
}
|
||||
var planes [][]*types.Issue
|
||||
if len(regular) > 0 {
|
||||
planes = append(planes, regular)
|
||||
}
|
||||
if len(wisp) > 0 {
|
||||
planes = append(planes, wisp)
|
||||
}
|
||||
return planes
|
||||
}
|
||||
|
||||
// orderImportIssuesForChunking returns the issues reordered so that every valid
|
||||
// readiness-affecting edge (blocks, parent-child, conditional-blocks, waits-for;
|
||||
// the types GetReadyWork consults) points at a row in the same or an earlier
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue