feat: add watchdog recovery decisions
This commit is contained in:
@@ -231,6 +231,27 @@ function recomputeStatus(payload) {
|
||||
return 'active';
|
||||
}
|
||||
|
||||
function decideRecoveryAction(payload, status) {
|
||||
if (!payload || typeof payload !== 'object') {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (status !== 'done_but_not_forwarded') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const attemptCountRaw = payload.recoveryAttemptCount;
|
||||
const recoveryAttemptCount = Number.isFinite(attemptCountRaw)
|
||||
? attemptCountRaw
|
||||
: Number.parseInt(String(attemptCountRaw ?? '0'), 10);
|
||||
|
||||
if (!Number.isNaN(recoveryAttemptCount) && recoveryAttemptCount >= 1) {
|
||||
return 'respawn';
|
||||
}
|
||||
|
||||
return 'fetch_history';
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
|
||||
@@ -244,6 +265,7 @@ function main() {
|
||||
const dispatchWrite = writeDispatchReceiptState(inputPayload);
|
||||
const completionWrite = writeCompletionReceiptState(inputPayload);
|
||||
const status = recomputeStatus(inputPayload);
|
||||
const recoveryDecision = decideRecoveryAction(inputPayload, status);
|
||||
|
||||
if ('content' in input) {
|
||||
delete input.content;
|
||||
@@ -272,6 +294,7 @@ function main() {
|
||||
message: status === 'not_implemented'
|
||||
? 'Dispatch and completion receipt writes are implemented; status recompute only handles basic active/suspect/completed states.'
|
||||
: 'Basic watchdog status recompute completed.',
|
||||
recoveryDecision,
|
||||
records,
|
||||
dispatchReceiptWrite: dispatchWrite,
|
||||
completionReceiptWrite: completionWrite,
|
||||
|
||||
@@ -202,6 +202,72 @@ ${result.stderr}`);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
|
||||
test('watchdog prefers fetch_history recovery when child run is done but no forwarded completion receipt exists', () => {
|
||||
const runner = createFixtureRunner();
|
||||
|
||||
try {
|
||||
const inputPath = runner.writeFixture('dispatch-done-not-forwarded-recovery.json', {
|
||||
runId: 'fixture-run-done-not-forwarded-recovery',
|
||||
childSessionKey: 'session:done-not-forwarded-recovery',
|
||||
dispatchAt: '2026-04-24T10:00:00.000Z',
|
||||
expectedBy: '2026-04-24T10:10:00.000Z',
|
||||
currentTime: '2026-04-24T10:05:00.000Z',
|
||||
childRunStatus: 'done',
|
||||
forwardedToMain: false,
|
||||
recoveryAttemptCount: 0,
|
||||
});
|
||||
|
||||
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
||||
|
||||
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}
|
||||
${result.stderr}`);
|
||||
assert.equal(result.stderr, '');
|
||||
|
||||
const payload = JSON.parse(result.stdout);
|
||||
assert.equal(payload.ok, true);
|
||||
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
||||
assert.equal(payload.result.recoveryDecision, 'fetch_history');
|
||||
} finally {
|
||||
runner.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
|
||||
test('watchdog escalates to respawn when fetch_history recovery was already attempted and delivery is still not forwarded', () => {
|
||||
const runner = createFixtureRunner();
|
||||
|
||||
try {
|
||||
const inputPath = runner.writeFixture('dispatch-done-not-forwarded-respawn.json', {
|
||||
runId: 'fixture-run-done-not-forwarded-respawn',
|
||||
childSessionKey: 'session:done-not-forwarded-respawn',
|
||||
dispatchAt: '2026-04-24T10:00:00.000Z',
|
||||
expectedBy: '2026-04-24T10:10:00.000Z',
|
||||
currentTime: '2026-04-24T10:06:00.000Z',
|
||||
childRunStatus: 'done',
|
||||
forwardedToMain: false,
|
||||
recoveryAttemptCount: 1,
|
||||
recoveryAction: 'fetch_history',
|
||||
lastRecoveryAt: '2026-04-24T10:05:30.000Z',
|
||||
});
|
||||
|
||||
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
||||
|
||||
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}
|
||||
${result.stderr}`);
|
||||
assert.equal(result.stderr, '');
|
||||
|
||||
const payload = JSON.parse(result.stdout);
|
||||
assert.equal(payload.ok, true);
|
||||
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
||||
assert.equal(payload.result.recoveryDecision, 'respawn');
|
||||
} finally {
|
||||
runner.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test('fixture runner exposes missing-input behavior for future fail-first cases', () => {
|
||||
const runner = createFixtureRunner();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user