Tame executor containment and expose arm control
All checks were successful
deploy / deploy (push) Successful in 33s
All checks were successful
deploy / deploy (push) Successful in 33s
Proof: Sentinel auto-containment now ignores quote-stale-only conditions, executor arming is exposed in the dashboard safe controls, and regression tests cover both containment gating and control routing. Assumptions: Quote silence alone is not sufficient evidence of a broken execution truth path, while ingest disconnect, publish stall, and history-writer stall remain containment-grade failures. Still fake: Live deployment still depends on the repo pipeline completing for this commit; executor disarm provenance is still not durably explained in the dashboard.
This commit is contained in:
parent
28a4a7ea6c
commit
96b5e3cbe7
5 changed files with 59 additions and 10 deletions
|
|
@ -13,6 +13,7 @@ import {
|
|||
buildRuntimeAlert,
|
||||
createRuntimeHealthThresholds,
|
||||
evaluateRuntimeHealth,
|
||||
shouldContainExecutorForAlerts,
|
||||
} from '../core/runtime-health.mjs';
|
||||
import {
|
||||
assertFundingObservationEvent,
|
||||
|
|
@ -447,15 +448,12 @@ function buildDeterministicRuntimeAlerts({ servicesByName, now, previousRuntimeE
|
|||
}
|
||||
|
||||
const executorArmed = executor?.state?.armed === true;
|
||||
const criticalTruthFailure = alerts.some((alert) => (
|
||||
alert.severity === 'critical'
|
||||
&& ['near_intents_ingest_disconnected', 'near_intents_quotes_stale', 'near_intents_publish_stale', 'history_writer_stalled'].includes(alert.alert_code)
|
||||
));
|
||||
const criticalTruthFailure = shouldContainExecutorForAlerts(alerts);
|
||||
if (executorArmed && criticalTruthFailure) {
|
||||
alerts.push(buildRuntimeAlert({
|
||||
alert_code: 'executor_armed_with_stale_truth',
|
||||
severity: 'critical',
|
||||
reason: 'trade-executor remains armed while upstream quote truth is critically stale',
|
||||
reason: 'trade-executor remains armed while the upstream truth path is critically broken',
|
||||
service_scope: 'trade-executor',
|
||||
pair: config.activePair,
|
||||
details: {
|
||||
|
|
@ -566,10 +564,7 @@ function buildAnomalyAlerts({ servicesByName, now }) {
|
|||
|
||||
async function maybeContainRisk({ servicesByName, desiredRuntimeAlerts, now }) {
|
||||
const executor = servicesByName['trade-executor'];
|
||||
const criticalTruthFailure = desiredRuntimeAlerts.some((alert) => (
|
||||
alert.severity === 'critical'
|
||||
&& ['near_intents_ingest_disconnected', 'near_intents_quotes_stale', 'near_intents_publish_stale', 'history_writer_stalled'].includes(alert.alert_code)
|
||||
));
|
||||
const criticalTruthFailure = shouldContainExecutorForAlerts(desiredRuntimeAlerts);
|
||||
const executorArmed = executor?.state?.armed === true;
|
||||
|
||||
if (!criticalTruthFailure) {
|
||||
|
|
|
|||
|
|
@ -125,6 +125,16 @@ const CONTROL_DEFINITIONS = [
|
|||
page: 'system',
|
||||
risk_class: 'safe',
|
||||
},
|
||||
{
|
||||
service: 'trade-executor',
|
||||
action: 'arm',
|
||||
method: 'POST',
|
||||
path: '/arm',
|
||||
label: 'Arm Executor',
|
||||
description: 'Persistently arm trade-executor so actionable decisions can submit.',
|
||||
page: 'system',
|
||||
risk_class: 'safe',
|
||||
},
|
||||
{
|
||||
service: 'trade-executor',
|
||||
action: 'pause',
|
||||
|
|
@ -762,7 +772,7 @@ function buildStrategySummary({ servicesByName, activeAlerts, recentTradeDecisio
|
|||
)),
|
||||
omitted_controls: [
|
||||
'Strategy arm and disarm are intentionally absent in this turn.',
|
||||
'Executor arm and drain are intentionally absent in this turn.',
|
||||
'Executor drain remains intentionally absent in this turn.',
|
||||
'Live withdrawal submission is intentionally absent in this turn.',
|
||||
],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -294,6 +294,18 @@ export function buildRuntimeAlert({
|
|||
};
|
||||
}
|
||||
|
||||
export function shouldContainExecutorForAlerts(alerts = []) {
|
||||
const containmentAlertCodes = new Set([
|
||||
'near_intents_ingest_disconnected',
|
||||
'near_intents_publish_stale',
|
||||
'history_writer_stalled',
|
||||
]);
|
||||
|
||||
return (alerts || []).some((alert) => (
|
||||
alert?.severity === 'critical' && containmentAlertCodes.has(alert.alert_code)
|
||||
));
|
||||
}
|
||||
|
||||
export function ageMs(value, now = new Date().toISOString()) {
|
||||
if (!value) return null;
|
||||
const left = new Date(value).getTime();
|
||||
|
|
|
|||
|
|
@ -105,6 +105,10 @@ test('control routing only resolves the allowlisted safe dashboard actions', ()
|
|||
service: 'liquidity-manager',
|
||||
action: 'refresh',
|
||||
});
|
||||
const armExecutor = resolveDashboardControl({
|
||||
service: 'trade-executor',
|
||||
action: 'arm',
|
||||
});
|
||||
const resumeExecutor = resolveDashboardControl({
|
||||
service: 'trade-executor',
|
||||
action: 'resume',
|
||||
|
|
@ -116,6 +120,8 @@ test('control routing only resolves the allowlisted safe dashboard actions', ()
|
|||
|
||||
assert.equal(refresh?.path, '/refresh');
|
||||
assert.equal(refresh?.risk_class, 'safe');
|
||||
assert.equal(armExecutor?.path, '/arm');
|
||||
assert.equal(armExecutor?.label, 'Arm Executor');
|
||||
assert.equal(resumeExecutor?.path, '/resume');
|
||||
assert.equal(resumeExecutor?.label, 'Resume Executor Intake');
|
||||
assert.equal(risky, null);
|
||||
|
|
|
|||
26
test/runtime-health.test.mjs
Normal file
26
test/runtime-health.test.mjs
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { shouldContainExecutorForAlerts } from '../src/core/runtime-health.mjs';
|
||||
|
||||
test('executor containment ignores quote-stale-only conditions', () => {
|
||||
assert.equal(shouldContainExecutorForAlerts([{
|
||||
alert_code: 'near_intents_quotes_stale',
|
||||
severity: 'critical',
|
||||
}]), false);
|
||||
});
|
||||
|
||||
test('executor containment still triggers on broken truth path alerts', () => {
|
||||
assert.equal(shouldContainExecutorForAlerts([{
|
||||
alert_code: 'near_intents_ingest_disconnected',
|
||||
severity: 'critical',
|
||||
}]), true);
|
||||
assert.equal(shouldContainExecutorForAlerts([{
|
||||
alert_code: 'near_intents_publish_stale',
|
||||
severity: 'critical',
|
||||
}]), true);
|
||||
assert.equal(shouldContainExecutorForAlerts([{
|
||||
alert_code: 'history_writer_stalled',
|
||||
severity: 'critical',
|
||||
}]), true);
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue