434 lines
13 KiB
JavaScript
Executable File
434 lines
13 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import process from 'node:process';
|
|
import crypto from 'node:crypto';
|
|
|
|
const ROOT_DIR = path.resolve(import.meta.dirname, '..');
|
|
const DEFAULT_STATE_PATH = path.join(ROOT_DIR, 'memory', 'watchdog-state.json');
|
|
const DEFAULT_EVIDENCE_DIR = path.join(ROOT_DIR, 'state', 'long-task-watchdog');
|
|
const DEFAULT_EVENT_DIR = path.join(ROOT_DIR, 'state', 'long-task-watchdog-events');
|
|
const DEFAULT_NOTIFICATION_DIR = path.join(ROOT_DIR, 'state', 'operator-notify-queue');
|
|
|
|
function parseArgs(argv) {
|
|
const args = {
|
|
compact: false,
|
|
state: DEFAULT_STATE_PATH,
|
|
now: null,
|
|
evidenceDir: DEFAULT_EVIDENCE_DIR,
|
|
eventDir: DEFAULT_EVENT_DIR,
|
|
notificationDir: DEFAULT_NOTIFICATION_DIR,
|
|
writeState: false,
|
|
help: false,
|
|
};
|
|
|
|
for (let i = 0; i < argv.length; i += 1) {
|
|
const token = argv[i];
|
|
if (token === '--compact') {
|
|
args.compact = true;
|
|
continue;
|
|
}
|
|
if (token === '--write-state') {
|
|
args.writeState = true;
|
|
continue;
|
|
}
|
|
if (token === '--help' || token === '-h') {
|
|
args.help = true;
|
|
continue;
|
|
}
|
|
if (token === '--state') {
|
|
args.state = argv[i + 1] ?? args.state;
|
|
i += 1;
|
|
continue;
|
|
}
|
|
if (token.startsWith('--state=')) {
|
|
args.state = token.slice('--state='.length) || args.state;
|
|
continue;
|
|
}
|
|
if (token === '--now') {
|
|
args.now = argv[i + 1] ?? null;
|
|
i += 1;
|
|
continue;
|
|
}
|
|
if (token.startsWith('--now=')) {
|
|
args.now = token.slice('--now='.length) || null;
|
|
continue;
|
|
}
|
|
if (token === '--evidence-dir') {
|
|
args.evidenceDir = argv[i + 1] ?? args.evidenceDir;
|
|
i += 1;
|
|
continue;
|
|
}
|
|
if (token.startsWith('--evidence-dir=')) {
|
|
args.evidenceDir = token.slice('--evidence-dir='.length) || args.evidenceDir;
|
|
continue;
|
|
}
|
|
if (token === '--event-dir') {
|
|
args.eventDir = argv[i + 1] ?? args.eventDir;
|
|
i += 1;
|
|
continue;
|
|
}
|
|
if (token.startsWith('--event-dir=')) {
|
|
args.eventDir = token.slice('--event-dir='.length) || args.eventDir;
|
|
continue;
|
|
}
|
|
if (token === '--notification-dir') {
|
|
args.notificationDir = argv[i + 1] ?? args.notificationDir;
|
|
i += 1;
|
|
continue;
|
|
}
|
|
if (token.startsWith('--notification-dir=')) {
|
|
args.notificationDir = token.slice('--notification-dir='.length) || args.notificationDir;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
return args;
|
|
}
|
|
|
|
function printHelp() {
|
|
process.stdout.write([
|
|
'Usage: node scripts/long_task_watchdog.mjs [--compact] [--write-state] [--state <path>] [--now <iso>] [--evidence-dir <path>] [--event-dir <path>] [--notification-dir <path>]',
|
|
'',
|
|
'Minimal file-backed long-task watchdog runner with operator-notification queue output.',
|
|
].join('\n') + '\n');
|
|
}
|
|
|
|
function parseJsonFile(filePath) {
|
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
return JSON.parse(raw);
|
|
}
|
|
|
|
function parseTime(value) {
|
|
if (typeof value !== 'string' || value.length === 0) return null;
|
|
const timestamp = Date.parse(value);
|
|
return Number.isNaN(timestamp) ? null : timestamp;
|
|
}
|
|
|
|
function toIso(value) {
|
|
return new Date(value).toISOString();
|
|
}
|
|
|
|
function toSafeName(value) {
|
|
return String(value || 'watchdog')
|
|
.replace(/[^a-zA-Z0-9._-]+/g, '-')
|
|
.replace(/^-+|-+$/g, '')
|
|
.slice(0, 80) || 'watchdog';
|
|
}
|
|
|
|
function makeEventId(prefix) {
|
|
return `${prefix}_${crypto.randomUUID()}`;
|
|
}
|
|
|
|
function evaluateWatchdog(watchdog, nowMs) {
|
|
const intervalMinutes = Number.isFinite(watchdog?.intervalMinutes)
|
|
? watchdog.intervalMinutes
|
|
: Number.parseInt(String(watchdog?.intervalMinutes ?? '0'), 10);
|
|
const intervalMs = intervalMinutes > 0 ? intervalMinutes * 60 * 1000 : 0;
|
|
const milestoneMs = parseTime(watchdog?.lastMilestoneAt);
|
|
const lastAlertMs = parseTime(watchdog?.lastAlertAt);
|
|
const active = watchdog?.status === 'active';
|
|
|
|
if (!active) {
|
|
return {
|
|
id: watchdog?.id ?? null,
|
|
active: false,
|
|
overdue: false,
|
|
action: 'skip_inactive',
|
|
reason: 'watchdog is not active',
|
|
};
|
|
}
|
|
|
|
if (!intervalMs || milestoneMs === null) {
|
|
return {
|
|
id: watchdog?.id ?? null,
|
|
active: true,
|
|
overdue: false,
|
|
action: 'invalid_contract',
|
|
reason: 'intervalMinutes or lastMilestoneAt is missing/invalid',
|
|
};
|
|
}
|
|
|
|
const dueAtMs = milestoneMs + intervalMs;
|
|
const overdue = nowMs >= dueAtMs;
|
|
|
|
if (!overdue) {
|
|
return {
|
|
id: watchdog?.id ?? null,
|
|
active: true,
|
|
overdue: false,
|
|
action: 'within_interval',
|
|
reason: 'last milestone is still within interval',
|
|
dueAt: toIso(dueAtMs),
|
|
minutesOverdue: 0,
|
|
};
|
|
}
|
|
|
|
const lastAlertStillFresh = lastAlertMs !== null && lastAlertMs >= dueAtMs;
|
|
if (lastAlertStillFresh) {
|
|
return {
|
|
id: watchdog?.id ?? null,
|
|
active: true,
|
|
overdue: true,
|
|
action: 'already_alerted_this_interval',
|
|
reason: 'lastAlertAt already covers current overdue interval',
|
|
dueAt: toIso(dueAtMs),
|
|
minutesOverdue: Math.floor((nowMs - dueAtMs) / 60000),
|
|
};
|
|
}
|
|
|
|
return {
|
|
id: watchdog?.id ?? null,
|
|
active: true,
|
|
overdue: true,
|
|
action: 'emit_external_evidence',
|
|
reason: 'active watchdog is overdue and has not been externally evidenced for this interval',
|
|
dueAt: toIso(dueAtMs),
|
|
minutesOverdue: Math.floor((nowMs - dueAtMs) / 60000),
|
|
};
|
|
}
|
|
|
|
function ensureDir(dirPath) {
|
|
fs.mkdirSync(dirPath, { recursive: true });
|
|
}
|
|
|
|
function writeJsonFile(dirPath, fileName, payload) {
|
|
ensureDir(dirPath);
|
|
const filePath = path.join(dirPath, fileName);
|
|
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
|
|
return filePath;
|
|
}
|
|
|
|
function buildBaseRefs(evidencePath) {
|
|
return [
|
|
{
|
|
kind: 'runtime_artifact',
|
|
path: evidencePath,
|
|
label: 'watchdog_evidence',
|
|
},
|
|
];
|
|
}
|
|
|
|
function writeEvidence(evidenceDir, watchdog, evaluation, nowIso) {
|
|
const fileName = `${nowIso.replace(/[:]/g, '').replace(/\.\d{3}Z$/, 'Z')}-${toSafeName(watchdog.id)}.json`;
|
|
const payload = {
|
|
generatedAt: nowIso,
|
|
tool: 'long_task_watchdog',
|
|
watchdog: {
|
|
id: watchdog.id,
|
|
task: watchdog.task,
|
|
ownerSession: watchdog.ownerSession ?? null,
|
|
ownerSessionKey: watchdog.ownerSessionKey ?? null,
|
|
reportChannel: watchdog.reportChannel ?? watchdog.channel ?? null,
|
|
reportTarget: watchdog.reportTarget ?? watchdog.target ?? null,
|
|
intervalMinutes: watchdog.intervalMinutes,
|
|
lastMilestoneAt: watchdog.lastMilestoneAt ?? null,
|
|
lastAlertAt: watchdog.lastAlertAt ?? null,
|
|
},
|
|
evaluation,
|
|
nextExpectedExternalAction: [
|
|
'nudge owner session',
|
|
'report owner-visible checkpoint',
|
|
'or respawn / inspect locally if owner appears stalled',
|
|
],
|
|
};
|
|
return writeJsonFile(evidenceDir, fileName, payload);
|
|
}
|
|
|
|
function buildWatchdogEvent(watchdog, evaluation, nowIso, evidencePath) {
|
|
const eventId = makeEventId('evt');
|
|
return {
|
|
event_id: eventId,
|
|
event_type: 'watchdog_fired',
|
|
runtime: 'openclaw',
|
|
adapter_version: '1.1.0',
|
|
agent_id: watchdog.ownerSessionKey ?? watchdog.ownerAgentId ?? 'agent:unknown',
|
|
task_id: watchdog.id,
|
|
correlation_id: `watchdog:${watchdog.id}`,
|
|
timestamp: nowIso,
|
|
payload: {
|
|
watchdog_type: 'long_task_overdue',
|
|
trigger_reason: evaluation.reason,
|
|
triggered_at: nowIso,
|
|
policy_id: 'long-task-watchdog-overdue-v1',
|
|
severity: 'critical',
|
|
due_at: evaluation.dueAt ?? null,
|
|
minutes_overdue: evaluation.minutesOverdue ?? null,
|
|
},
|
|
evidence_refs: buildBaseRefs(evidencePath),
|
|
operator_context: {
|
|
channel: watchdog.reportChannel ?? watchdog.channel ?? null,
|
|
operator_id: watchdog.reportTarget ?? watchdog.target ?? null,
|
|
reporting_mode: 'watchdog',
|
|
silent_task: true,
|
|
watchdog_policy_id: 'long-task-watchdog-overdue-v1',
|
|
},
|
|
};
|
|
}
|
|
|
|
function writeEvent(eventDir, watchdog, nowIso, event) {
|
|
const fileName = `${nowIso.replace(/[:]/g, '').replace(/\.\d{3}Z$/, 'Z')}-${toSafeName(watchdog.id)}-watchdog-fired.json`;
|
|
return writeJsonFile(eventDir, fileName, event);
|
|
}
|
|
|
|
function buildNotificationMessage(watchdog, evaluation, nowIso) {
|
|
const lines = [
|
|
'【Watchdog 逾時告警】',
|
|
`task: ${watchdog.task ?? watchdog.id ?? 'unknown-task'}`,
|
|
`watchdog: ${watchdog.id ?? 'unknown-watchdog'}`,
|
|
`dueAt: ${evaluation.dueAt ?? 'unknown'}`,
|
|
`minutesOverdue: ${evaluation.minutesOverdue ?? 'unknown'}`,
|
|
`lastMilestoneAt: ${watchdog.lastMilestoneAt ?? 'unknown'}`,
|
|
`triggeredAt: ${nowIso}`,
|
|
`ownerSessionKey: ${watchdog.ownerSessionKey ?? watchdog.ownerSession ?? 'unknown'}`,
|
|
'requiredAction: 請立即對 owner/operator 發出可見更新,或檢查/重派 stalled task。',
|
|
];
|
|
return lines.join('\n');
|
|
}
|
|
|
|
function buildNotificationQueueItem(watchdog, evaluation, nowIso, evidencePath, eventPath, eventId) {
|
|
const notificationId = makeEventId('notify');
|
|
const channel = watchdog.reportChannel ?? watchdog.channel ?? null;
|
|
const target = watchdog.reportTarget ?? watchdog.target ?? null;
|
|
return {
|
|
notification_id: notificationId,
|
|
kind: 'notify_operator',
|
|
status: 'pending',
|
|
created_at: nowIso,
|
|
source_tool: 'long_task_watchdog',
|
|
severity: 'critical',
|
|
operator_notice: {
|
|
required: true,
|
|
channel,
|
|
target,
|
|
urgency: 'critical',
|
|
message: buildNotificationMessage(watchdog, evaluation, nowIso),
|
|
must_reference: ['watchdog_fired', 'forced_operator_update'],
|
|
},
|
|
dispatch_hint: {
|
|
tool: 'message.send',
|
|
channel,
|
|
target,
|
|
message: buildNotificationMessage(watchdog, evaluation, nowIso),
|
|
},
|
|
governance: {
|
|
task_id: watchdog.id,
|
|
correlation_id: `watchdog:${watchdog.id}`,
|
|
trigger_event_id: eventId,
|
|
trigger_event_type: 'watchdog_fired',
|
|
decision: 'force_checkpoint',
|
|
policy_id: 'long-task-watchdog-overdue-v1',
|
|
reason: evaluation.reason,
|
|
},
|
|
evidence_refs: [
|
|
...buildBaseRefs(evidencePath),
|
|
{
|
|
kind: 'runtime_artifact',
|
|
path: eventPath,
|
|
label: 'watchdog_event',
|
|
},
|
|
],
|
|
blocked_gap: channel && target
|
|
? null
|
|
: 'watchdog state does not define reportChannel/reportTarget, so dispatcher target is incomplete',
|
|
};
|
|
}
|
|
|
|
function writeNotificationQueueItem(notificationDir, watchdog, nowIso, queueItem) {
|
|
const fileName = `${nowIso.replace(/[:]/g, '').replace(/\.\d{3}Z$/, 'Z')}-${toSafeName(watchdog.id)}-notify-operator.json`;
|
|
return writeJsonFile(notificationDir, fileName, queueItem);
|
|
}
|
|
|
|
function main() {
|
|
const args = parseArgs(process.argv.slice(2));
|
|
if (args.help) {
|
|
printHelp();
|
|
process.exit(0);
|
|
}
|
|
|
|
const nowMs = args.now ? parseTime(args.now) : Date.now();
|
|
if (nowMs === null) {
|
|
process.stderr.write('Invalid --now value\n');
|
|
process.exit(1);
|
|
}
|
|
const nowIso = toIso(nowMs);
|
|
|
|
const state = parseJsonFile(args.state);
|
|
const watchdogs = Array.isArray(state.watchdogs) ? state.watchdogs : [];
|
|
const evaluations = watchdogs.map((watchdog) => ({
|
|
watchdogId: watchdog?.id ?? null,
|
|
...evaluateWatchdog(watchdog, nowMs),
|
|
}));
|
|
|
|
const evidenceWrites = [];
|
|
const eventWrites = [];
|
|
const notificationWrites = [];
|
|
const nextWatchdogs = watchdogs.map((watchdog, index) => {
|
|
const evaluation = evaluations[index];
|
|
if (evaluation.action !== 'emit_external_evidence') {
|
|
return watchdog;
|
|
}
|
|
const evidencePath = writeEvidence(args.evidenceDir, watchdog, evaluation, nowIso);
|
|
evidenceWrites.push({ watchdogId: watchdog.id, path: evidencePath });
|
|
|
|
const event = buildWatchdogEvent(watchdog, evaluation, nowIso, evidencePath);
|
|
const eventPath = writeEvent(args.eventDir, watchdog, nowIso, event);
|
|
eventWrites.push({ watchdogId: watchdog.id, path: eventPath, eventId: event.event_id, eventType: event.event_type });
|
|
|
|
const notification = buildNotificationQueueItem(watchdog, evaluation, nowIso, evidencePath, eventPath, event.event_id);
|
|
const notificationPath = writeNotificationQueueItem(args.notificationDir, watchdog, nowIso, notification);
|
|
notificationWrites.push({
|
|
watchdogId: watchdog.id,
|
|
path: notificationPath,
|
|
notificationId: notification.notification_id,
|
|
channel: notification.operator_notice.channel,
|
|
target: notification.operator_notice.target,
|
|
dispatchReady: notification.blocked_gap === null,
|
|
blockedGap: notification.blocked_gap,
|
|
});
|
|
|
|
return {
|
|
...watchdog,
|
|
lastAlertAt: nowIso,
|
|
lastObservedActivityAt: watchdog.lastObservedActivityAt ?? watchdog.lastMilestoneAt ?? null,
|
|
lastNudgeAt: nowIso,
|
|
};
|
|
});
|
|
|
|
if (args.writeState) {
|
|
const nextState = {
|
|
...state,
|
|
watchdogs: nextWatchdogs,
|
|
};
|
|
fs.writeFileSync(args.state, `${JSON.stringify(nextState, null, 2)}\n`, 'utf8');
|
|
}
|
|
|
|
const response = {
|
|
ok: true,
|
|
tool: 'long_task_watchdog',
|
|
version: 'mvp-v2',
|
|
statePath: path.resolve(args.state),
|
|
evidenceDir: path.resolve(args.evidenceDir),
|
|
eventDir: path.resolve(args.eventDir),
|
|
notificationDir: path.resolve(args.notificationDir),
|
|
now: nowIso,
|
|
writeState: args.writeState,
|
|
result: {
|
|
activeCount: watchdogs.filter((item) => item?.status === 'active').length,
|
|
overdueCount: evaluations.filter((item) => item.overdue === true).length,
|
|
emittedCount: evidenceWrites.length,
|
|
eventCount: eventWrites.length,
|
|
notificationCount: notificationWrites.length,
|
|
evaluations,
|
|
evidenceWrites,
|
|
eventWrites,
|
|
notificationWrites,
|
|
},
|
|
};
|
|
|
|
process.stdout.write(`${JSON.stringify(response, null, args.compact ? 0 : 2)}\n`);
|
|
}
|
|
|
|
main();
|