feat: add watchdog recovery decisions

This commit is contained in:
Eve
2026-04-24 15:08:53 +08:00
parent 6b1592f066
commit 73f47cfdf7
2 changed files with 89 additions and 0 deletions

View File

@@ -231,6 +231,27 @@ function recomputeStatus(payload) {
return 'active';
}
function decideRecoveryAction(payload, status) {
if (!payload || typeof payload !== 'object') {
return null;
}
if (status !== 'done_but_not_forwarded') {
return null;
}
const attemptCountRaw = payload.recoveryAttemptCount;
const recoveryAttemptCount = Number.isFinite(attemptCountRaw)
? attemptCountRaw
: Number.parseInt(String(attemptCountRaw ?? '0'), 10);
if (!Number.isNaN(recoveryAttemptCount) && recoveryAttemptCount >= 1) {
return 'respawn';
}
return 'fetch_history';
}
function main() {
const args = parseArgs(process.argv.slice(2));
@@ -244,6 +265,7 @@ function main() {
const dispatchWrite = writeDispatchReceiptState(inputPayload);
const completionWrite = writeCompletionReceiptState(inputPayload);
const status = recomputeStatus(inputPayload);
const recoveryDecision = decideRecoveryAction(inputPayload, status);
if ('content' in input) {
delete input.content;
@@ -272,6 +294,7 @@ function main() {
message: status === 'not_implemented'
? 'Dispatch and completion receipt writes are implemented; status recompute only handles basic active/suspect/completed states.'
: 'Basic watchdog status recompute completed.',
recoveryDecision,
records,
dispatchReceiptWrite: dispatchWrite,
completionReceiptWrite: completionWrite,

View File

@@ -202,6 +202,72 @@ ${result.stderr}`);
}
});
test('watchdog prefers fetch_history recovery when child run is done but no forwarded completion receipt exists', () => {
const runner = createFixtureRunner();
try {
const inputPath = runner.writeFixture('dispatch-done-not-forwarded-recovery.json', {
runId: 'fixture-run-done-not-forwarded-recovery',
childSessionKey: 'session:done-not-forwarded-recovery',
dispatchAt: '2026-04-24T10:00:00.000Z',
expectedBy: '2026-04-24T10:10:00.000Z',
currentTime: '2026-04-24T10:05:00.000Z',
childRunStatus: 'done',
forwardedToMain: false,
recoveryAttemptCount: 0,
});
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}
${result.stderr}`);
assert.equal(result.stderr, '');
const payload = JSON.parse(result.stdout);
assert.equal(payload.ok, true);
assert.equal(payload.result.status, 'done_but_not_forwarded');
assert.equal(payload.result.recoveryDecision, 'fetch_history');
} finally {
runner.cleanup();
}
});
test('watchdog escalates to respawn when fetch_history recovery was already attempted and delivery is still not forwarded', () => {
const runner = createFixtureRunner();
try {
const inputPath = runner.writeFixture('dispatch-done-not-forwarded-respawn.json', {
runId: 'fixture-run-done-not-forwarded-respawn',
childSessionKey: 'session:done-not-forwarded-respawn',
dispatchAt: '2026-04-24T10:00:00.000Z',
expectedBy: '2026-04-24T10:10:00.000Z',
currentTime: '2026-04-24T10:06:00.000Z',
childRunStatus: 'done',
forwardedToMain: false,
recoveryAttemptCount: 1,
recoveryAction: 'fetch_history',
lastRecoveryAt: '2026-04-24T10:05:30.000Z',
});
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}
${result.stderr}`);
assert.equal(result.stderr, '');
const payload = JSON.parse(result.stdout);
assert.equal(payload.ok, true);
assert.equal(payload.result.status, 'done_but_not_forwarded');
assert.equal(payload.result.recoveryDecision, 'respawn');
} finally {
runner.cleanup();
}
});
test('fixture runner exposes missing-input behavior for future fail-first cases', () => {
const runner = createFixtureRunner();