Skip to content

Commit d7e01ce

Browse files
authored
test(action-ledger): make cleanup-stuck projection tests robust under host load (#709)
* test(action-ledger): make cleanup-stuck projection tests robust under host load The scoped-rejection test gave the independent root a 1000ms CrabFleet timeout, which also bounds projection queue wait. On a loaded host the 5ms polling loops can consume more than 1s of wall clock before the test frees a slot, so the queued independent projection settled with 'queue timed out after 1000ms' and could never start. Raise it to 30s and widen the 500ms poll deadlines to 5s. On failure the test also leaked its stuck cleanup resolvers, permanently holding 3 of 4 global projection slots and cascading into the next test ('hold slots until response cleanup settles' saw started 1 !== 4). Release the resolvers in a finally block so a failure stays scoped. * test(action-ledger): release late-registered cleanup resolvers in failure teardown Autoreview: after the finally block drains the resolver array, a queued blocked post can still start and register a new stuck cleanup on a later microtask, hanging the flush. Gate the mock on a release flag so any cancel after teardown begins resolves immediately.
1 parent 3beb5f0 commit d7e01ce

1 file changed

Lines changed: 31 additions & 21 deletions

File tree

test/action-ledger-runtime.test.ts

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3254,18 +3254,22 @@ test("cleanup-stuck projection rejection is scoped to the affected spool root",
32543254
const independentEnv = workflowEnv({
32553255
CLAWSWEEPER_CRABFLEET_AGENT_TOKEN: "agent-token",
32563256
CLAWSWEEPER_CRABFLEET_SESSION_ID: "session-1",
3257-
CLAWSWEEPER_CRABFLEET_TIMEOUT_MS: "1000",
3257+
// Also bounds queue wait; must outlast the polling below on a loaded host.
3258+
CLAWSWEEPER_CRABFLEET_TIMEOUT_MS: "30000",
32583259
});
32593260
const cleanupResolvers: Array<() => void> = [];
3261+
let releaseCleanups = false;
32603262
const blockedFetch = (async () =>
32613263
({
32623264
ok: true,
32633265
status: 204,
32643266
body: {
32653267
cancel: () =>
3266-
new Promise<void>((resolve) => {
3267-
cleanupResolvers.push(resolve);
3268-
}),
3268+
releaseCleanups
3269+
? Promise.resolve()
3270+
: new Promise<void>((resolve) => {
3271+
cleanupResolvers.push(resolve);
3272+
}),
32693273
},
32703274
}) as unknown as Response) as typeof fetch;
32713275
for (let index = 0; index < CRABFLEET_PROJECTION_LIMITS.maxConcurrent + 1; index += 1) {
@@ -3279,26 +3283,32 @@ test("cleanup-stuck projection rejection is scoped to the affected spool root",
32793283
}) as typeof fetch),
32803284
);
32813285

3282-
const blockedDeadline = Date.now() + 500;
3283-
while (cleanupResolvers.length < CRABFLEET_PROJECTION_LIMITS.maxConcurrent) {
3284-
if (Date.now() >= blockedDeadline) {
3285-
throw new Error("blocked root did not enter response cleanup");
3286+
try {
3287+
const blockedDeadline = Date.now() + 5000;
3288+
while (cleanupResolvers.length < CRABFLEET_PROJECTION_LIMITS.maxConcurrent) {
3289+
if (Date.now() >= blockedDeadline) {
3290+
throw new Error("blocked root did not enter response cleanup");
3291+
}
3292+
await new Promise((resolve) => setTimeout(resolve, 5));
32863293
}
3287-
await new Promise((resolve) => setTimeout(resolve, 5));
3288-
}
3289-
await new Promise((resolve) => setTimeout(resolve, 20));
3290-
assert.equal(independentStarted, 0);
3291-
3292-
cleanupResolvers.shift()!();
3293-
const independentDeadline = Date.now() + 500;
3294-
while (independentStarted === 0) {
3295-
if (Date.now() >= independentDeadline) {
3296-
throw new Error("independent root did not start after a projection slot recovered");
3294+
await new Promise((resolve) => setTimeout(resolve, 20));
3295+
assert.equal(independentStarted, 0);
3296+
3297+
cleanupResolvers.shift()!();
3298+
const independentDeadline = Date.now() + 5000;
3299+
while (independentStarted === 0) {
3300+
if (Date.now() >= independentDeadline) {
3301+
throw new Error("independent root did not start after a projection slot recovered");
3302+
}
3303+
await new Promise((resolve) => setTimeout(resolve, 5));
32973304
}
3298-
await new Promise((resolve) => setTimeout(resolve, 5));
3305+
} finally {
3306+
// Release held slots even on failure so later tests do not inherit a starved pool.
3307+
// The flag makes any cleanup registered after this point resolve immediately.
3308+
releaseCleanups = true;
3309+
while (cleanupResolvers.length > 0) cleanupResolvers.shift()!();
3310+
await flushPendingCrabFleetPosts();
32993311
}
3300-
for (const resolve of cleanupResolvers) resolve();
3301-
await flushPendingCrabFleetPosts();
33023312
assert.equal(independentStarted, 1);
33033313
});
33043314

0 commit comments

Comments
 (0)