553 lines
19 KiB
JavaScript
553 lines
19 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import assert from 'node:assert/strict';
|
|
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import path from 'node:path';
|
|
import process from 'node:process';
|
|
import { spawnSync } from 'node:child_process';
|
|
|
|
const ROOT_DIR = path.resolve(import.meta.dirname, '..');
|
|
const WATCHDOG_SCRIPT = path.join(ROOT_DIR, 'scripts', 'subagent_delivery_watchdog.mjs');
|
|
|
|
function createFixtureRunner() {
|
|
const fixtureRoot = mkdtempSync(path.join(tmpdir(), 'subagent-watchdog-test-'));
|
|
|
|
function writeFixture(name, content) {
|
|
const fixturePath = path.join(fixtureRoot, name);
|
|
const body = typeof content === 'string' ? content : JSON.stringify(content, null, 2);
|
|
writeFileSync(fixturePath, body);
|
|
return fixturePath;
|
|
}
|
|
|
|
function runWatchdog(args = [], options = {}) {
|
|
const result = spawnSync(process.execPath, [WATCHDOG_SCRIPT, ...args], {
|
|
cwd: ROOT_DIR,
|
|
encoding: 'utf8',
|
|
...options,
|
|
});
|
|
|
|
return {
|
|
status: result.status,
|
|
signal: result.signal,
|
|
stdout: result.stdout ?? '',
|
|
stderr: result.stderr ?? '',
|
|
error: result.error ?? null,
|
|
};
|
|
}
|
|
|
|
function cleanup() {
|
|
rmSync(fixtureRoot, { recursive: true, force: true });
|
|
}
|
|
|
|
return {
|
|
fixtureRoot,
|
|
writeFixture,
|
|
runWatchdog,
|
|
cleanup,
|
|
};
|
|
}
|
|
|
|
const tests = [];
|
|
|
|
function test(name, fn) {
|
|
tests.push({ name, fn });
|
|
}
|
|
|
|
function printResult(prefix, name, detail = '') {
|
|
const suffix = detail ? ` ${detail}` : '';
|
|
process.stdout.write(`${prefix} ${name}${suffix}\n`);
|
|
}
|
|
|
|
function runFixture(payloadInput, fixtureName = 'fixture.json') {
|
|
const runner = createFixtureRunner();
|
|
|
|
try {
|
|
const inputPath = runner.writeFixture(fixtureName, payloadInput);
|
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
|
|
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}\n${result.stderr}`);
|
|
assert.equal(result.stderr, '');
|
|
|
|
return {
|
|
payload: JSON.parse(result.stdout),
|
|
inputPath,
|
|
};
|
|
} finally {
|
|
runner.cleanup();
|
|
}
|
|
}
|
|
|
|
test('fixture runner can invoke watchdog skeleton with a generated input file', () => {
|
|
const runner = createFixtureRunner();
|
|
|
|
try {
|
|
const inputPath = runner.writeFixture('dispatch.json', {
|
|
runId: 'fixture-run-001',
|
|
childSessionKey: 'session:test',
|
|
});
|
|
|
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
|
|
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}\n${result.stderr}`);
|
|
assert.equal(result.stderr, '');
|
|
|
|
const payload = JSON.parse(result.stdout);
|
|
assert.equal(payload.ok, true);
|
|
assert.equal(payload.tool, 'subagent_delivery_watchdog');
|
|
assert.equal(payload.result.status, 'not_implemented');
|
|
assert.equal(payload.input.path, inputPath);
|
|
assert.equal(payload.input.exists, true);
|
|
} finally {
|
|
runner.cleanup();
|
|
}
|
|
});
|
|
|
|
test('watchdog reports active before SLA when dispatch exists and no completion receipt has arrived yet', () => {
|
|
const runner = createFixtureRunner();
|
|
|
|
try {
|
|
const inputPath = runner.writeFixture('dispatch-before-sla.json', {
|
|
runId: 'fixture-run-active-before-sla',
|
|
childSessionKey: 'session:active-before-sla',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:05:00.000Z',
|
|
});
|
|
|
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
|
|
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}\n${result.stderr}`);
|
|
assert.equal(result.stderr, '');
|
|
|
|
const payload = JSON.parse(result.stdout);
|
|
assert.equal(payload.ok, true);
|
|
assert.equal(payload.input.path, inputPath);
|
|
assert.equal(payload.input.exists, true);
|
|
assert.equal(payload.result.status, 'active');
|
|
} finally {
|
|
runner.cleanup();
|
|
}
|
|
});
|
|
|
|
test('watchdog reports suspect delivery failure after SLA when dispatch exists and no completion receipt has arrived yet', () => {
|
|
const runner = createFixtureRunner();
|
|
|
|
try {
|
|
const inputPath = runner.writeFixture('dispatch-beyond-sla.json', {
|
|
runId: 'fixture-run-suspect-delivery-failure',
|
|
childSessionKey: 'session:suspect-delivery-failure',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:15:00.000Z',
|
|
});
|
|
|
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
|
|
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}\n${result.stderr}`);
|
|
assert.equal(result.stderr, '');
|
|
|
|
const payload = JSON.parse(result.stdout);
|
|
assert.equal(payload.ok, true);
|
|
assert.equal(payload.input.path, inputPath);
|
|
assert.equal(payload.input.exists, true);
|
|
assert.equal(payload.result.status, 'suspect_delivery_failure');
|
|
} finally {
|
|
runner.cleanup();
|
|
}
|
|
});
|
|
|
|
test('watchdog reports completed when dispatch exists and completion receipt has arrived', () => {
|
|
const runner = createFixtureRunner();
|
|
|
|
try {
|
|
const inputPath = runner.writeFixture('dispatch-completed.json', {
|
|
runId: 'fixture-run-completed',
|
|
childSessionKey: 'session:completed',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:05:00.000Z',
|
|
completionReceiptAt: '2026-04-24T10:04:00.000Z',
|
|
});
|
|
|
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
|
|
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}\n${result.stderr}`);
|
|
assert.equal(result.stderr, '');
|
|
|
|
const payload = JSON.parse(result.stdout);
|
|
assert.equal(payload.ok, true);
|
|
assert.equal(payload.input.path, inputPath);
|
|
assert.equal(payload.input.exists, true);
|
|
assert.equal(payload.result.status, 'completed');
|
|
} finally {
|
|
runner.cleanup();
|
|
}
|
|
});
|
|
|
|
test('watchdog reports done but not forwarded when child run is marked done without a main-thread completion receipt', () => {
|
|
const runner = createFixtureRunner();
|
|
|
|
try {
|
|
const inputPath = runner.writeFixture('dispatch-done-not-forwarded.json', {
|
|
runId: 'fixture-run-done-not-forwarded',
|
|
childSessionKey: 'session:done-not-forwarded',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:05:00.000Z',
|
|
childRunStatus: 'done',
|
|
});
|
|
|
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
|
|
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}\n${result.stderr}`);
|
|
assert.equal(result.stderr, '');
|
|
|
|
const payload = JSON.parse(result.stdout);
|
|
assert.equal(payload.ok, true);
|
|
assert.equal(payload.input.path, inputPath);
|
|
assert.equal(payload.input.exists, true);
|
|
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
|
} finally {
|
|
runner.cleanup();
|
|
}
|
|
});
|
|
|
|
test('watchdog prefers fetch_history recovery when child run is done but no forwarded completion receipt exists', () => {
|
|
const runner = createFixtureRunner();
|
|
|
|
try {
|
|
const inputPath = runner.writeFixture('dispatch-done-not-forwarded-recovery.json', {
|
|
runId: 'fixture-run-done-not-forwarded-recovery',
|
|
childSessionKey: 'session:done-not-forwarded-recovery',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:05:00.000Z',
|
|
childRunStatus: 'done',
|
|
forwardedToMain: false,
|
|
recoveryAttemptCount: 0,
|
|
});
|
|
|
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
|
|
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}\n${result.stderr}`);
|
|
assert.equal(result.stderr, '');
|
|
|
|
const payload = JSON.parse(result.stdout);
|
|
assert.equal(payload.ok, true);
|
|
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
|
assert.equal(payload.result.recoveryDecision, 'fetch_history');
|
|
} finally {
|
|
runner.cleanup();
|
|
}
|
|
});
|
|
|
|
test('watchdog escalates to respawn when fetch_history recovery was already attempted and delivery is still not forwarded', () => {
|
|
const runner = createFixtureRunner();
|
|
|
|
try {
|
|
const inputPath = runner.writeFixture('dispatch-done-not-forwarded-respawn.json', {
|
|
runId: 'fixture-run-done-not-forwarded-respawn',
|
|
childSessionKey: 'session:done-not-forwarded-respawn',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:06:00.000Z',
|
|
childRunStatus: 'done',
|
|
forwardedToMain: false,
|
|
recoveryAttemptCount: 1,
|
|
recoveryAction: 'fetch_history',
|
|
lastRecoveryAt: '2026-04-24T10:05:30.000Z',
|
|
});
|
|
|
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
|
|
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}\n${result.stderr}`);
|
|
assert.equal(result.stderr, '');
|
|
|
|
const payload = JSON.parse(result.stdout);
|
|
assert.equal(payload.ok, true);
|
|
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
|
assert.equal(payload.result.recoveryDecision, 'respawn');
|
|
} finally {
|
|
runner.cleanup();
|
|
}
|
|
});
|
|
|
|
test('watchdog escalates to blocked when respawn recovery was already attempted and delivery is still not forwarded', () => {
|
|
const runner = createFixtureRunner();
|
|
|
|
try {
|
|
const inputPath = runner.writeFixture('dispatch-done-not-forwarded-blocked.json', {
|
|
runId: 'fixture-run-done-not-forwarded-blocked',
|
|
childSessionKey: 'session:done-not-forwarded-blocked',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:07:00.000Z',
|
|
childRunStatus: 'done',
|
|
forwardedToMain: false,
|
|
recoveryAttemptCount: 2,
|
|
recoveryAction: 'respawn',
|
|
lastRecoveryAt: '2026-04-24T10:06:30.000Z',
|
|
});
|
|
|
|
const result = runner.runWatchdog(['--compact', '--input', inputPath]);
|
|
|
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}\n${result.stderr}`);
|
|
assert.equal(result.stderr, '');
|
|
|
|
const payload = JSON.parse(result.stdout);
|
|
assert.equal(payload.ok, true);
|
|
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
|
assert.equal(payload.result.recoveryDecision, 'blocked');
|
|
} finally {
|
|
runner.cleanup();
|
|
}
|
|
});
|
|
|
|
test('owner-visible reporting marks suspect_delivery_failure as reportable with minimal payload', () => {
|
|
const { payload } = runFixture({
|
|
runId: 'fixture-run-report-suspect',
|
|
childSessionKey: 'session:report-suspect',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:15:00.000Z',
|
|
}, 'report-suspect.json');
|
|
|
|
assert.deepEqual(payload.result.reporting, {
|
|
ownerVisible: true,
|
|
category: 'suspect_delivery_failure',
|
|
decision: 'report',
|
|
summary: 'Subagent delivery is suspected to have failed after crossing SLA.',
|
|
detail: {
|
|
runId: 'fixture-run-report-suspect',
|
|
childSessionKey: 'session:report-suspect',
|
|
status: 'suspect_delivery_failure',
|
|
recoveryDecision: null,
|
|
},
|
|
});
|
|
});
|
|
|
|
test('owner-visible reporting marks done_but_not_forwarded plus fetch_history as reportable with minimal payload', () => {
|
|
const { payload } = runFixture({
|
|
runId: 'fixture-run-report-fetch-history',
|
|
childSessionKey: 'session:report-fetch-history',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:05:00.000Z',
|
|
childRunStatus: 'done',
|
|
recoveryAttemptCount: 0,
|
|
}, 'report-fetch-history.json');
|
|
|
|
assert.deepEqual(payload.result.reporting, {
|
|
ownerVisible: true,
|
|
category: 'done_but_not_forwarded',
|
|
decision: 'fetch_history',
|
|
summary: 'Child run is done but no forwarded completion receipt is confirmed yet.',
|
|
detail: {
|
|
runId: 'fixture-run-report-fetch-history',
|
|
childSessionKey: 'session:report-fetch-history',
|
|
status: 'done_but_not_forwarded',
|
|
recoveryDecision: 'fetch_history',
|
|
},
|
|
});
|
|
});
|
|
|
|
test('owner-visible reporting marks respawn as reportable with minimal payload', () => {
|
|
const { payload } = runFixture({
|
|
runId: 'fixture-run-report-respawn',
|
|
childSessionKey: 'session:report-respawn',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:06:00.000Z',
|
|
childRunStatus: 'done',
|
|
recoveryAttemptCount: 1,
|
|
recoveryAction: 'fetch_history',
|
|
}, 'report-respawn.json');
|
|
|
|
assert.deepEqual(payload.result.reporting, {
|
|
ownerVisible: true,
|
|
category: 'done_but_not_forwarded',
|
|
decision: 'respawn',
|
|
summary: 'Child run is done but recovery already failed once; respawn is the next conservative step.',
|
|
detail: {
|
|
runId: 'fixture-run-report-respawn',
|
|
childSessionKey: 'session:report-respawn',
|
|
status: 'done_but_not_forwarded',
|
|
recoveryDecision: 'respawn',
|
|
},
|
|
});
|
|
});
|
|
|
|
test('owner-visible reporting marks blocked as reportable with minimal payload', () => {
|
|
const { payload } = runFixture({
|
|
runId: 'fixture-run-report-blocked',
|
|
childSessionKey: 'session:report-blocked',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:07:00.000Z',
|
|
childRunStatus: 'done',
|
|
recoveryAttemptCount: 2,
|
|
recoveryAction: 'respawn',
|
|
}, 'report-blocked.json');
|
|
|
|
assert.deepEqual(payload.result.reporting, {
|
|
ownerVisible: true,
|
|
category: 'done_but_not_forwarded',
|
|
decision: 'blocked',
|
|
summary: 'Child run is still not forwarded after repeated recovery attempts; owner attention is required.',
|
|
detail: {
|
|
runId: 'fixture-run-report-blocked',
|
|
childSessionKey: 'session:report-blocked',
|
|
status: 'done_but_not_forwarded',
|
|
recoveryDecision: 'blocked',
|
|
},
|
|
});
|
|
});
|
|
|
|
test('scenario matrix: normal completion stays non-owner-visible and carries no recovery action', () => {
|
|
const { payload } = runFixture({
|
|
runId: 'fixture-scenario-normal-completion',
|
|
childSessionKey: 'session:scenario-normal-completion',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:05:00.000Z',
|
|
completionReceiptAt: '2026-04-24T10:04:00.000Z',
|
|
forwardedToMain: true,
|
|
}, 'scenario-normal-completion.json');
|
|
|
|
assert.equal(payload.result.status, 'completed');
|
|
assert.equal(payload.result.recoveryDecision, null);
|
|
assert.deepEqual(payload.result.reporting, {
|
|
ownerVisible: false,
|
|
category: 'completed',
|
|
decision: 'none',
|
|
summary: 'Completion receipt is present; no owner-visible report is needed.',
|
|
detail: {
|
|
runId: 'fixture-scenario-normal-completion',
|
|
childSessionKey: 'session:scenario-normal-completion',
|
|
status: 'completed',
|
|
recoveryDecision: null,
|
|
},
|
|
});
|
|
});
|
|
|
|
test('scenario matrix: slow-but-active stays non-owner-visible before SLA', () => {
|
|
const { payload } = runFixture({
|
|
runId: 'fixture-scenario-slow-but-active',
|
|
childSessionKey: 'session:scenario-slow-but-active',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:09:59.000Z',
|
|
}, 'scenario-slow-but-active.json');
|
|
|
|
assert.equal(payload.result.status, 'active');
|
|
assert.equal(payload.result.recoveryDecision, null);
|
|
assert.deepEqual(payload.result.reporting, {
|
|
ownerVisible: false,
|
|
category: 'active',
|
|
decision: 'none',
|
|
summary: 'Dispatch is still within SLA; no owner-visible report is needed.',
|
|
detail: {
|
|
runId: 'fixture-scenario-slow-but-active',
|
|
childSessionKey: 'session:scenario-slow-but-active',
|
|
status: 'active',
|
|
recoveryDecision: null,
|
|
},
|
|
});
|
|
});
|
|
|
|
test('scenario matrix: done-but-not-forwarded resolves to fetch_history reporting decision', () => {
|
|
const { payload } = runFixture({
|
|
runId: 'fixture-scenario-fetch-history',
|
|
childSessionKey: 'session:scenario-fetch-history',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:05:00.000Z',
|
|
childRunStatus: 'done',
|
|
recoveryAttemptCount: 0,
|
|
}, 'scenario-fetch-history.json');
|
|
|
|
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
|
assert.equal(payload.result.recoveryDecision, 'fetch_history');
|
|
assert.equal(payload.result.reporting.ownerVisible, true);
|
|
assert.equal(payload.result.reporting.decision, 'fetch_history');
|
|
});
|
|
|
|
test('scenario matrix: repeated failure escalates to respawn reporting decision', () => {
|
|
const { payload } = runFixture({
|
|
runId: 'fixture-scenario-respawn',
|
|
childSessionKey: 'session:scenario-respawn',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:06:00.000Z',
|
|
childRunStatus: 'done',
|
|
recoveryAttemptCount: 1,
|
|
recoveryAction: 'fetch_history',
|
|
}, 'scenario-respawn.json');
|
|
|
|
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
|
assert.equal(payload.result.recoveryDecision, 'respawn');
|
|
assert.equal(payload.result.reporting.ownerVisible, true);
|
|
assert.equal(payload.result.reporting.decision, 'respawn');
|
|
});
|
|
|
|
test('scenario matrix: repeated failure can escalate to blocked reporting decision', () => {
|
|
const { payload } = runFixture({
|
|
runId: 'fixture-scenario-blocked',
|
|
childSessionKey: 'session:scenario-blocked',
|
|
dispatchAt: '2026-04-24T10:00:00.000Z',
|
|
expectedBy: '2026-04-24T10:10:00.000Z',
|
|
currentTime: '2026-04-24T10:07:00.000Z',
|
|
childRunStatus: 'done',
|
|
recoveryAttemptCount: 2,
|
|
recoveryAction: 'respawn',
|
|
}, 'scenario-blocked.json');
|
|
|
|
assert.equal(payload.result.status, 'done_but_not_forwarded');
|
|
assert.equal(payload.result.recoveryDecision, 'blocked');
|
|
assert.equal(payload.result.reporting.ownerVisible, true);
|
|
assert.equal(payload.result.reporting.decision, 'blocked');
|
|
});
|
|
|
|
test('fixture runner exposes missing-input behavior for future fail-first cases', () => {
|
|
const runner = createFixtureRunner();
|
|
|
|
try {
|
|
const missingPath = path.join(runner.fixtureRoot, 'missing.json');
|
|
const result = runner.runWatchdog(['--compact', '--input', missingPath]);
|
|
|
|
assert.equal(result.status, 0, `expected zero exit status, got ${result.status}\n${result.stderr}`);
|
|
|
|
const payload = JSON.parse(result.stdout);
|
|
assert.equal(payload.ok, true);
|
|
assert.equal(payload.input.path, missingPath);
|
|
assert.equal(payload.input.exists, false);
|
|
assert.equal(payload.result.status, 'not_implemented');
|
|
} finally {
|
|
runner.cleanup();
|
|
}
|
|
});
|
|
|
|
function main() {
|
|
let passed = 0;
|
|
|
|
for (const { name, fn } of tests) {
|
|
try {
|
|
fn();
|
|
passed += 1;
|
|
printResult('PASS', name);
|
|
} catch (error) {
|
|
printResult('FAIL', name, error instanceof Error ? `- ${error.message}` : `- ${String(error)}`);
|
|
if (error instanceof Error && error.stack) {
|
|
process.stderr.write(`${error.stack}\n`);
|
|
}
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
const failed = tests.length - passed;
|
|
process.stdout.write(`\nSummary: ${passed} passed, ${failed} failed, ${tests.length} total\n`);
|
|
}
|
|
|
|
main();
|