feat(reporting-governance): add package-first portability smoke
This commit is contained in:
Eve
433
plugins/reporting-governance/scripts/long_task_watchdog.mjs
Executable file
433
plugins/reporting-governance/scripts/long_task_watchdog.mjs
Executable file
@@ -0,0 +1,433 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
const ROOT_DIR = path.resolve(import.meta.dirname, '..');
|
||||
const DEFAULT_STATE_PATH = path.join(ROOT_DIR, 'memory', 'watchdog-state.json');
|
||||
const DEFAULT_EVIDENCE_DIR = path.join(ROOT_DIR, 'state', 'long-task-watchdog');
|
||||
const DEFAULT_EVENT_DIR = path.join(ROOT_DIR, 'state', 'long-task-watchdog-events');
|
||||
const DEFAULT_NOTIFICATION_DIR = path.join(ROOT_DIR, 'state', 'operator-notify-queue');
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
compact: false,
|
||||
state: DEFAULT_STATE_PATH,
|
||||
now: null,
|
||||
evidenceDir: DEFAULT_EVIDENCE_DIR,
|
||||
eventDir: DEFAULT_EVENT_DIR,
|
||||
notificationDir: DEFAULT_NOTIFICATION_DIR,
|
||||
writeState: false,
|
||||
help: false,
|
||||
};
|
||||
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const token = argv[i];
|
||||
if (token === '--compact') {
|
||||
args.compact = true;
|
||||
continue;
|
||||
}
|
||||
if (token === '--write-state') {
|
||||
args.writeState = true;
|
||||
continue;
|
||||
}
|
||||
if (token === '--help' || token === '-h') {
|
||||
args.help = true;
|
||||
continue;
|
||||
}
|
||||
if (token === '--state') {
|
||||
args.state = argv[i + 1] ?? args.state;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (token.startsWith('--state=')) {
|
||||
args.state = token.slice('--state='.length) || args.state;
|
||||
continue;
|
||||
}
|
||||
if (token === '--now') {
|
||||
args.now = argv[i + 1] ?? null;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (token.startsWith('--now=')) {
|
||||
args.now = token.slice('--now='.length) || null;
|
||||
continue;
|
||||
}
|
||||
if (token === '--evidence-dir') {
|
||||
args.evidenceDir = argv[i + 1] ?? args.evidenceDir;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (token.startsWith('--evidence-dir=')) {
|
||||
args.evidenceDir = token.slice('--evidence-dir='.length) || args.evidenceDir;
|
||||
continue;
|
||||
}
|
||||
if (token === '--event-dir') {
|
||||
args.eventDir = argv[i + 1] ?? args.eventDir;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (token.startsWith('--event-dir=')) {
|
||||
args.eventDir = token.slice('--event-dir='.length) || args.eventDir;
|
||||
continue;
|
||||
}
|
||||
if (token === '--notification-dir') {
|
||||
args.notificationDir = argv[i + 1] ?? args.notificationDir;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (token.startsWith('--notification-dir=')) {
|
||||
args.notificationDir = token.slice('--notification-dir='.length) || args.notificationDir;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
process.stdout.write([
|
||||
'Usage: node scripts/long_task_watchdog.mjs [--compact] [--write-state] [--state <path>] [--now <iso>] [--evidence-dir <path>] [--event-dir <path>] [--notification-dir <path>]',
|
||||
'',
|
||||
'Minimal file-backed long-task watchdog runner with operator-notification queue output.',
|
||||
].join('\n') + '\n');
|
||||
}
|
||||
|
||||
function parseJsonFile(filePath) {
|
||||
const raw = fs.readFileSync(filePath, 'utf8');
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
|
||||
function parseTime(value) {
|
||||
if (typeof value !== 'string' || value.length === 0) return null;
|
||||
const timestamp = Date.parse(value);
|
||||
return Number.isNaN(timestamp) ? null : timestamp;
|
||||
}
|
||||
|
||||
function toIso(value) {
|
||||
return new Date(value).toISOString();
|
||||
}
|
||||
|
||||
function toSafeName(value) {
|
||||
return String(value || 'watchdog')
|
||||
.replace(/[^a-zA-Z0-9._-]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.slice(0, 80) || 'watchdog';
|
||||
}
|
||||
|
||||
function makeEventId(prefix) {
|
||||
return `${prefix}_${crypto.randomUUID()}`;
|
||||
}
|
||||
|
||||
function evaluateWatchdog(watchdog, nowMs) {
|
||||
const intervalMinutes = Number.isFinite(watchdog?.intervalMinutes)
|
||||
? watchdog.intervalMinutes
|
||||
: Number.parseInt(String(watchdog?.intervalMinutes ?? '0'), 10);
|
||||
const intervalMs = intervalMinutes > 0 ? intervalMinutes * 60 * 1000 : 0;
|
||||
const milestoneMs = parseTime(watchdog?.lastMilestoneAt);
|
||||
const lastAlertMs = parseTime(watchdog?.lastAlertAt);
|
||||
const active = watchdog?.status === 'active';
|
||||
|
||||
if (!active) {
|
||||
return {
|
||||
id: watchdog?.id ?? null,
|
||||
active: false,
|
||||
overdue: false,
|
||||
action: 'skip_inactive',
|
||||
reason: 'watchdog is not active',
|
||||
};
|
||||
}
|
||||
|
||||
if (!intervalMs || milestoneMs === null) {
|
||||
return {
|
||||
id: watchdog?.id ?? null,
|
||||
active: true,
|
||||
overdue: false,
|
||||
action: 'invalid_contract',
|
||||
reason: 'intervalMinutes or lastMilestoneAt is missing/invalid',
|
||||
};
|
||||
}
|
||||
|
||||
const dueAtMs = milestoneMs + intervalMs;
|
||||
const overdue = nowMs >= dueAtMs;
|
||||
|
||||
if (!overdue) {
|
||||
return {
|
||||
id: watchdog?.id ?? null,
|
||||
active: true,
|
||||
overdue: false,
|
||||
action: 'within_interval',
|
||||
reason: 'last milestone is still within interval',
|
||||
dueAt: toIso(dueAtMs),
|
||||
minutesOverdue: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const lastAlertStillFresh = lastAlertMs !== null && lastAlertMs >= dueAtMs;
|
||||
if (lastAlertStillFresh) {
|
||||
return {
|
||||
id: watchdog?.id ?? null,
|
||||
active: true,
|
||||
overdue: true,
|
||||
action: 'already_alerted_this_interval',
|
||||
reason: 'lastAlertAt already covers current overdue interval',
|
||||
dueAt: toIso(dueAtMs),
|
||||
minutesOverdue: Math.floor((nowMs - dueAtMs) / 60000),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
id: watchdog?.id ?? null,
|
||||
active: true,
|
||||
overdue: true,
|
||||
action: 'emit_external_evidence',
|
||||
reason: 'active watchdog is overdue and has not been externally evidenced for this interval',
|
||||
dueAt: toIso(dueAtMs),
|
||||
minutesOverdue: Math.floor((nowMs - dueAtMs) / 60000),
|
||||
};
|
||||
}
|
||||
|
||||
function ensureDir(dirPath) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
function writeJsonFile(dirPath, fileName, payload) {
|
||||
ensureDir(dirPath);
|
||||
const filePath = path.join(dirPath, fileName);
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
|
||||
return filePath;
|
||||
}
|
||||
|
||||
function buildBaseRefs(evidencePath) {
|
||||
return [
|
||||
{
|
||||
kind: 'runtime_artifact',
|
||||
path: evidencePath,
|
||||
label: 'watchdog_evidence',
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function writeEvidence(evidenceDir, watchdog, evaluation, nowIso) {
|
||||
const fileName = `${nowIso.replace(/[:]/g, '').replace(/\.\d{3}Z$/, 'Z')}-${toSafeName(watchdog.id)}.json`;
|
||||
const payload = {
|
||||
generatedAt: nowIso,
|
||||
tool: 'long_task_watchdog',
|
||||
watchdog: {
|
||||
id: watchdog.id,
|
||||
task: watchdog.task,
|
||||
ownerSession: watchdog.ownerSession ?? null,
|
||||
ownerSessionKey: watchdog.ownerSessionKey ?? null,
|
||||
reportChannel: watchdog.reportChannel ?? watchdog.channel ?? null,
|
||||
reportTarget: watchdog.reportTarget ?? watchdog.target ?? null,
|
||||
intervalMinutes: watchdog.intervalMinutes,
|
||||
lastMilestoneAt: watchdog.lastMilestoneAt ?? null,
|
||||
lastAlertAt: watchdog.lastAlertAt ?? null,
|
||||
},
|
||||
evaluation,
|
||||
nextExpectedExternalAction: [
|
||||
'nudge owner session',
|
||||
'report owner-visible checkpoint',
|
||||
'or respawn / inspect locally if owner appears stalled',
|
||||
],
|
||||
};
|
||||
return writeJsonFile(evidenceDir, fileName, payload);
|
||||
}
|
||||
|
||||
function buildWatchdogEvent(watchdog, evaluation, nowIso, evidencePath) {
|
||||
const eventId = makeEventId('evt');
|
||||
return {
|
||||
event_id: eventId,
|
||||
event_type: 'watchdog_fired',
|
||||
runtime: 'openclaw',
|
||||
adapter_version: '1.1.0',
|
||||
agent_id: watchdog.ownerSessionKey ?? watchdog.ownerAgentId ?? 'agent:unknown',
|
||||
task_id: watchdog.id,
|
||||
correlation_id: `watchdog:${watchdog.id}`,
|
||||
timestamp: nowIso,
|
||||
payload: {
|
||||
watchdog_type: 'long_task_overdue',
|
||||
trigger_reason: evaluation.reason,
|
||||
triggered_at: nowIso,
|
||||
policy_id: 'long-task-watchdog-overdue-v1',
|
||||
severity: 'critical',
|
||||
due_at: evaluation.dueAt ?? null,
|
||||
minutes_overdue: evaluation.minutesOverdue ?? null,
|
||||
},
|
||||
evidence_refs: buildBaseRefs(evidencePath),
|
||||
operator_context: {
|
||||
channel: watchdog.reportChannel ?? watchdog.channel ?? null,
|
||||
operator_id: watchdog.reportTarget ?? watchdog.target ?? null,
|
||||
reporting_mode: 'watchdog',
|
||||
silent_task: true,
|
||||
watchdog_policy_id: 'long-task-watchdog-overdue-v1',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function writeEvent(eventDir, watchdog, nowIso, event) {
|
||||
const fileName = `${nowIso.replace(/[:]/g, '').replace(/\.\d{3}Z$/, 'Z')}-${toSafeName(watchdog.id)}-watchdog-fired.json`;
|
||||
return writeJsonFile(eventDir, fileName, event);
|
||||
}
|
||||
|
||||
function buildNotificationMessage(watchdog, evaluation, nowIso) {
|
||||
const lines = [
|
||||
'【Watchdog 逾時告警】',
|
||||
`task: ${watchdog.task ?? watchdog.id ?? 'unknown-task'}`,
|
||||
`watchdog: ${watchdog.id ?? 'unknown-watchdog'}`,
|
||||
`dueAt: ${evaluation.dueAt ?? 'unknown'}`,
|
||||
`minutesOverdue: ${evaluation.minutesOverdue ?? 'unknown'}`,
|
||||
`lastMilestoneAt: ${watchdog.lastMilestoneAt ?? 'unknown'}`,
|
||||
`triggeredAt: ${nowIso}`,
|
||||
`ownerSessionKey: ${watchdog.ownerSessionKey ?? watchdog.ownerSession ?? 'unknown'}`,
|
||||
'requiredAction: 請立即對 owner/operator 發出可見更新，或檢查/重派 stalled task。',
|
||||
];
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function buildNotificationQueueItem(watchdog, evaluation, nowIso, evidencePath, eventPath, eventId) {
|
||||
const notificationId = makeEventId('notify');
|
||||
const channel = watchdog.reportChannel ?? watchdog.channel ?? null;
|
||||
const target = watchdog.reportTarget ?? watchdog.target ?? null;
|
||||
return {
|
||||
notification_id: notificationId,
|
||||
kind: 'notify_operator',
|
||||
status: 'pending',
|
||||
created_at: nowIso,
|
||||
source_tool: 'long_task_watchdog',
|
||||
severity: 'critical',
|
||||
operator_notice: {
|
||||
required: true,
|
||||
channel,
|
||||
target,
|
||||
urgency: 'critical',
|
||||
message: buildNotificationMessage(watchdog, evaluation, nowIso),
|
||||
must_reference: ['watchdog_fired', 'forced_operator_update'],
|
||||
},
|
||||
dispatch_hint: {
|
||||
tool: 'message.send',
|
||||
channel,
|
||||
target,
|
||||
message: buildNotificationMessage(watchdog, evaluation, nowIso),
|
||||
},
|
||||
governance: {
|
||||
task_id: watchdog.id,
|
||||
correlation_id: `watchdog:${watchdog.id}`,
|
||||
trigger_event_id: eventId,
|
||||
trigger_event_type: 'watchdog_fired',
|
||||
decision: 'force_checkpoint',
|
||||
policy_id: 'long-task-watchdog-overdue-v1',
|
||||
reason: evaluation.reason,
|
||||
},
|
||||
evidence_refs: [
|
||||
...buildBaseRefs(evidencePath),
|
||||
{
|
||||
kind: 'runtime_artifact',
|
||||
path: eventPath,
|
||||
label: 'watchdog_event',
|
||||
},
|
||||
],
|
||||
blocked_gap: channel && target
|
||||
? null
|
||||
: 'watchdog state does not define reportChannel/reportTarget, so dispatcher target is incomplete',
|
||||
};
|
||||
}
|
||||
|
||||
function writeNotificationQueueItem(notificationDir, watchdog, nowIso, queueItem) {
|
||||
const fileName = `${nowIso.replace(/[:]/g, '').replace(/\.\d{3}Z$/, 'Z')}-${toSafeName(watchdog.id)}-notify-operator.json`;
|
||||
return writeJsonFile(notificationDir, fileName, queueItem);
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (args.help) {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const nowMs = args.now ? parseTime(args.now) : Date.now();
|
||||
if (nowMs === null) {
|
||||
process.stderr.write('Invalid --now value\n');
|
||||
process.exit(1);
|
||||
}
|
||||
const nowIso = toIso(nowMs);
|
||||
|
||||
const state = parseJsonFile(args.state);
|
||||
const watchdogs = Array.isArray(state.watchdogs) ? state.watchdogs : [];
|
||||
const evaluations = watchdogs.map((watchdog) => ({
|
||||
watchdogId: watchdog?.id ?? null,
|
||||
...evaluateWatchdog(watchdog, nowMs),
|
||||
}));
|
||||
|
||||
const evidenceWrites = [];
|
||||
const eventWrites = [];
|
||||
const notificationWrites = [];
|
||||
const nextWatchdogs = watchdogs.map((watchdog, index) => {
|
||||
const evaluation = evaluations[index];
|
||||
if (evaluation.action !== 'emit_external_evidence') {
|
||||
return watchdog;
|
||||
}
|
||||
const evidencePath = writeEvidence(args.evidenceDir, watchdog, evaluation, nowIso);
|
||||
evidenceWrites.push({ watchdogId: watchdog.id, path: evidencePath });
|
||||
|
||||
const event = buildWatchdogEvent(watchdog, evaluation, nowIso, evidencePath);
|
||||
const eventPath = writeEvent(args.eventDir, watchdog, nowIso, event);
|
||||
eventWrites.push({ watchdogId: watchdog.id, path: eventPath, eventId: event.event_id, eventType: event.event_type });
|
||||
|
||||
const notification = buildNotificationQueueItem(watchdog, evaluation, nowIso, evidencePath, eventPath, event.event_id);
|
||||
const notificationPath = writeNotificationQueueItem(args.notificationDir, watchdog, nowIso, notification);
|
||||
notificationWrites.push({
|
||||
watchdogId: watchdog.id,
|
||||
path: notificationPath,
|
||||
notificationId: notification.notification_id,
|
||||
channel: notification.operator_notice.channel,
|
||||
target: notification.operator_notice.target,
|
||||
dispatchReady: notification.blocked_gap === null,
|
||||
blockedGap: notification.blocked_gap,
|
||||
});
|
||||
|
||||
return {
|
||||
...watchdog,
|
||||
lastAlertAt: nowIso,
|
||||
lastObservedActivityAt: watchdog.lastObservedActivityAt ?? watchdog.lastMilestoneAt ?? null,
|
||||
lastNudgeAt: nowIso,
|
||||
};
|
||||
});
|
||||
|
||||
if (args.writeState) {
|
||||
const nextState = {
|
||||
...state,
|
||||
watchdogs: nextWatchdogs,
|
||||
};
|
||||
fs.writeFileSync(args.state, `${JSON.stringify(nextState, null, 2)}\n`, 'utf8');
|
||||
}
|
||||
|
||||
const response = {
|
||||
ok: true,
|
||||
tool: 'long_task_watchdog',
|
||||
version: 'mvp-v2',
|
||||
statePath: path.resolve(args.state),
|
||||
evidenceDir: path.resolve(args.evidenceDir),
|
||||
eventDir: path.resolve(args.eventDir),
|
||||
notificationDir: path.resolve(args.notificationDir),
|
||||
now: nowIso,
|
||||
writeState: args.writeState,
|
||||
result: {
|
||||
activeCount: watchdogs.filter((item) => item?.status === 'active').length,
|
||||
overdueCount: evaluations.filter((item) => item.overdue === true).length,
|
||||
emittedCount: evidenceWrites.length,
|
||||
eventCount: eventWrites.length,
|
||||
notificationCount: notificationWrites.length,
|
||||
evaluations,
|
||||
evidenceWrites,
|
||||
eventWrites,
|
||||
notificationWrites,
|
||||
},
|
||||
};
|
||||
|
||||
process.stdout.write(`${JSON.stringify(response, null, args.compact ? 0 : 2)}\n`);
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user