feat: add watchdog recovery decisions
This commit is contained in:
@@ -231,6 +231,27 @@ function recomputeStatus(payload) {
|
|||||||
return 'active';
|
return 'active';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function decideRecoveryAction(payload, status) {
|
||||||
|
if (!payload || typeof payload !== 'object') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (status !== 'done_but_not_forwarded') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const attemptCountRaw = payload.recoveryAttemptCount;
|
||||||
|
const recoveryAttemptCount = Number.isFinite(attemptCountRaw)
|
||||||
|
? attemptCountRaw
|
||||||
|
: Number.parseInt(String(attemptCountRaw ?? '0'), 10);
|
||||||
|
|
||||||
|
if (!Number.isNaN(recoveryAttemptCount) && recoveryAttemptCount >= 1) {
|
||||||
|
return 'respawn';
|
||||||
|
}
|
||||||
|
|
||||||
|
return 'fetch_history';
|
||||||
|
}
|
||||||
|
|
||||||
function main() {
|
function main() {
|
||||||
const args = parseArgs(process.argv.slice(2));
|
const args = parseArgs(process.argv.slice(2));
|
||||||
|
|
||||||
@@ -244,6 +265,7 @@ function main() {
|
|||||||
const dispatchWrite = writeDispatchReceiptState(inputPayload);
|
const dispatchWrite = writeDispatchReceiptState(inputPayload);
|
||||||
const completionWrite = writeCompletionReceiptState(inputPayload);
|
const completionWrite = writeCompletionReceiptState(inputPayload);
|
||||||
const status = recomputeStatus(inputPayload);
|
const status = recomputeStatus(inputPayload);
|
||||||
|
const recoveryDecision = decideRecoveryAction(inputPayload, status);
|
||||||
|
|
||||||
if ('content' in input) {
|
if ('content' in input) {
|
||||||
delete input.content;
|
delete input.content;
|
||||||
@@ -272,6 +294,7 @@ function main() {
|
|||||||
message: status === 'not_implemented'
|
message: status === 'not_implemented'
|
||||||
? 'Dispatch and completion receipt writes are implemented; status recompute only handles basic active/suspect/completed states.'
|
? 'Dispatch and completion receipt writes are implemented; status recompute only handles basic active/suspect/completed states.'
|
||||||
: 'Basic watchdog status recompute completed.',
|
: 'Basic watchdog status recompute completed.',
|
||||||
|
recoveryDecision,
|
||||||
records,
|
records,
|
||||||
dispatchReceiptWrite: dispatchWrite,
|
dispatchReceiptWrite: dispatchWrite,
|
||||||
completionReceiptWrite: completionWrite,
|
completionReceiptWrite: completionWrite,
|
||||||
|
|||||||
@@ -202,6 +202,72 @@ ${result.stderr}`);
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
test('watchdog prefers fetch_history recovery when child run is done but no forwarded completion receipt exists', () => {
|
||||||
|
const runner = createFixtureRunner();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const inputPath = runner.writeFixture('dispatch-done-not-forwarded-recovery.json', {
|
||||||
|
runId: 'fixture-run-done-not-forwarded-recovery',
|
||||||
|
childSessionKey: 'session:done-not-forwarded-recovery',
|
||||||
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
||||||
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
||||||
|
currentTime: '2026-04-24T10:05:00.000Z',
|
||||||
|
childRunStatus: 'done',
|
||||||
|
forwardedToMain: false,
|
||||||
|
recoveryAttemptCount: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
||||||
|
|
||||||
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}
|
||||||
|
${result.stderr}`);
|
||||||
|
assert.equal(result.stderr, '');
|
||||||
|
|
||||||
|
const payload = JSON.parse(result.stdout);
|
||||||
|
assert.equal(payload.ok, true);
|
||||||
|
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
||||||
|
assert.equal(payload.result.recoveryDecision, 'fetch_history');
|
||||||
|
} finally {
|
||||||
|
runner.cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
test('watchdog escalates to respawn when fetch_history recovery was already attempted and delivery is still not forwarded', () => {
|
||||||
|
const runner = createFixtureRunner();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const inputPath = runner.writeFixture('dispatch-done-not-forwarded-respawn.json', {
|
||||||
|
runId: 'fixture-run-done-not-forwarded-respawn',
|
||||||
|
childSessionKey: 'session:done-not-forwarded-respawn',
|
||||||
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
||||||
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
||||||
|
currentTime: '2026-04-24T10:06:00.000Z',
|
||||||
|
childRunStatus: 'done',
|
||||||
|
forwardedToMain: false,
|
||||||
|
recoveryAttemptCount: 1,
|
||||||
|
recoveryAction: 'fetch_history',
|
||||||
|
lastRecoveryAt: '2026-04-24T10:05:30.000Z',
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
||||||
|
|
||||||
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}
|
||||||
|
${result.stderr}`);
|
||||||
|
assert.equal(result.stderr, '');
|
||||||
|
|
||||||
|
const payload = JSON.parse(result.stdout);
|
||||||
|
assert.equal(payload.ok, true);
|
||||||
|
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
||||||
|
assert.equal(payload.result.recoveryDecision, 'respawn');
|
||||||
|
} finally {
|
||||||
|
runner.cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('fixture runner exposes missing-input behavior for future fail-first cases', () => {
|
test('fixture runner exposes missing-input behavior for future fail-first cases', () => {
|
||||||
const runner = createFixtureRunner();
|
const runner = createFixtureRunner();
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user