Tame executor containment and expose arm control
All checks were successful
deploy / deploy (push) Successful in 33s
All checks were successful
deploy / deploy (push) Successful in 33s
Proof: Sentinel auto-containment now ignores quote-stale-only conditions, executor arming is exposed in the dashboard safe controls, and regression tests cover both containment gating and control routing. Assumptions: Quote silence alone is not sufficient evidence of a broken execution truth path, while ingest disconnect, publish stall, and history-writer stall remain containment-grade failures. Still fake: Live deployment still depends on the repo pipeline completing for this commit; executor disarm provenance is still not durably explained in the dashboard.
This commit is contained in:
parent
28a4a7ea6c
commit
96b5e3cbe7
5 changed files with 59 additions and 10 deletions
|
|
@ -13,6 +13,7 @@ import {
|
||||||
buildRuntimeAlert,
|
buildRuntimeAlert,
|
||||||
createRuntimeHealthThresholds,
|
createRuntimeHealthThresholds,
|
||||||
evaluateRuntimeHealth,
|
evaluateRuntimeHealth,
|
||||||
|
shouldContainExecutorForAlerts,
|
||||||
} from '../core/runtime-health.mjs';
|
} from '../core/runtime-health.mjs';
|
||||||
import {
|
import {
|
||||||
assertFundingObservationEvent,
|
assertFundingObservationEvent,
|
||||||
|
|
@ -447,15 +448,12 @@ function buildDeterministicRuntimeAlerts({ servicesByName, now, previousRuntimeE
|
||||||
}
|
}
|
||||||
|
|
||||||
const executorArmed = executor?.state?.armed === true;
|
const executorArmed = executor?.state?.armed === true;
|
||||||
const criticalTruthFailure = alerts.some((alert) => (
|
const criticalTruthFailure = shouldContainExecutorForAlerts(alerts);
|
||||||
alert.severity === 'critical'
|
|
||||||
&& ['near_intents_ingest_disconnected', 'near_intents_quotes_stale', 'near_intents_publish_stale', 'history_writer_stalled'].includes(alert.alert_code)
|
|
||||||
));
|
|
||||||
if (executorArmed && criticalTruthFailure) {
|
if (executorArmed && criticalTruthFailure) {
|
||||||
alerts.push(buildRuntimeAlert({
|
alerts.push(buildRuntimeAlert({
|
||||||
alert_code: 'executor_armed_with_stale_truth',
|
alert_code: 'executor_armed_with_stale_truth',
|
||||||
severity: 'critical',
|
severity: 'critical',
|
||||||
reason: 'trade-executor remains armed while upstream quote truth is critically stale',
|
reason: 'trade-executor remains armed while the upstream truth path is critically broken',
|
||||||
service_scope: 'trade-executor',
|
service_scope: 'trade-executor',
|
||||||
pair: config.activePair,
|
pair: config.activePair,
|
||||||
details: {
|
details: {
|
||||||
|
|
@ -566,10 +564,7 @@ function buildAnomalyAlerts({ servicesByName, now }) {
|
||||||
|
|
||||||
async function maybeContainRisk({ servicesByName, desiredRuntimeAlerts, now }) {
|
async function maybeContainRisk({ servicesByName, desiredRuntimeAlerts, now }) {
|
||||||
const executor = servicesByName['trade-executor'];
|
const executor = servicesByName['trade-executor'];
|
||||||
const criticalTruthFailure = desiredRuntimeAlerts.some((alert) => (
|
const criticalTruthFailure = shouldContainExecutorForAlerts(desiredRuntimeAlerts);
|
||||||
alert.severity === 'critical'
|
|
||||||
&& ['near_intents_ingest_disconnected', 'near_intents_quotes_stale', 'near_intents_publish_stale', 'history_writer_stalled'].includes(alert.alert_code)
|
|
||||||
));
|
|
||||||
const executorArmed = executor?.state?.armed === true;
|
const executorArmed = executor?.state?.armed === true;
|
||||||
|
|
||||||
if (!criticalTruthFailure) {
|
if (!criticalTruthFailure) {
|
||||||
|
|
|
||||||
|
|
@ -125,6 +125,16 @@ const CONTROL_DEFINITIONS = [
|
||||||
page: 'system',
|
page: 'system',
|
||||||
risk_class: 'safe',
|
risk_class: 'safe',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
service: 'trade-executor',
|
||||||
|
action: 'arm',
|
||||||
|
method: 'POST',
|
||||||
|
path: '/arm',
|
||||||
|
label: 'Arm Executor',
|
||||||
|
description: 'Persistently arm trade-executor so actionable decisions can submit.',
|
||||||
|
page: 'system',
|
||||||
|
risk_class: 'safe',
|
||||||
|
},
|
||||||
{
|
{
|
||||||
service: 'trade-executor',
|
service: 'trade-executor',
|
||||||
action: 'pause',
|
action: 'pause',
|
||||||
|
|
@ -762,7 +772,7 @@ function buildStrategySummary({ servicesByName, activeAlerts, recentTradeDecisio
|
||||||
)),
|
)),
|
||||||
omitted_controls: [
|
omitted_controls: [
|
||||||
'Strategy arm and disarm are intentionally absent in this turn.',
|
'Strategy arm and disarm are intentionally absent in this turn.',
|
||||||
'Executor arm and drain are intentionally absent in this turn.',
|
'Executor drain remains intentionally absent in this turn.',
|
||||||
'Live withdrawal submission is intentionally absent in this turn.',
|
'Live withdrawal submission is intentionally absent in this turn.',
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -294,6 +294,18 @@ export function buildRuntimeAlert({
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function shouldContainExecutorForAlerts(alerts = []) {
|
||||||
|
const containmentAlertCodes = new Set([
|
||||||
|
'near_intents_ingest_disconnected',
|
||||||
|
'near_intents_publish_stale',
|
||||||
|
'history_writer_stalled',
|
||||||
|
]);
|
||||||
|
|
||||||
|
return (alerts || []).some((alert) => (
|
||||||
|
alert?.severity === 'critical' && containmentAlertCodes.has(alert.alert_code)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
export function ageMs(value, now = new Date().toISOString()) {
|
export function ageMs(value, now = new Date().toISOString()) {
|
||||||
if (!value) return null;
|
if (!value) return null;
|
||||||
const left = new Date(value).getTime();
|
const left = new Date(value).getTime();
|
||||||
|
|
|
||||||
|
|
@ -105,6 +105,10 @@ test('control routing only resolves the allowlisted safe dashboard actions', ()
|
||||||
service: 'liquidity-manager',
|
service: 'liquidity-manager',
|
||||||
action: 'refresh',
|
action: 'refresh',
|
||||||
});
|
});
|
||||||
|
const armExecutor = resolveDashboardControl({
|
||||||
|
service: 'trade-executor',
|
||||||
|
action: 'arm',
|
||||||
|
});
|
||||||
const resumeExecutor = resolveDashboardControl({
|
const resumeExecutor = resolveDashboardControl({
|
||||||
service: 'trade-executor',
|
service: 'trade-executor',
|
||||||
action: 'resume',
|
action: 'resume',
|
||||||
|
|
@ -116,6 +120,8 @@ test('control routing only resolves the allowlisted safe dashboard actions', ()
|
||||||
|
|
||||||
assert.equal(refresh?.path, '/refresh');
|
assert.equal(refresh?.path, '/refresh');
|
||||||
assert.equal(refresh?.risk_class, 'safe');
|
assert.equal(refresh?.risk_class, 'safe');
|
||||||
|
assert.equal(armExecutor?.path, '/arm');
|
||||||
|
assert.equal(armExecutor?.label, 'Arm Executor');
|
||||||
assert.equal(resumeExecutor?.path, '/resume');
|
assert.equal(resumeExecutor?.path, '/resume');
|
||||||
assert.equal(resumeExecutor?.label, 'Resume Executor Intake');
|
assert.equal(resumeExecutor?.label, 'Resume Executor Intake');
|
||||||
assert.equal(risky, null);
|
assert.equal(risky, null);
|
||||||
|
|
|
||||||
26
test/runtime-health.test.mjs
Normal file
26
test/runtime-health.test.mjs
Normal file
|
|
@ -0,0 +1,26 @@
|
||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
import { shouldContainExecutorForAlerts } from '../src/core/runtime-health.mjs';
|
||||||
|
|
||||||
|
test('executor containment ignores quote-stale-only conditions', () => {
|
||||||
|
assert.equal(shouldContainExecutorForAlerts([{
|
||||||
|
alert_code: 'near_intents_quotes_stale',
|
||||||
|
severity: 'critical',
|
||||||
|
}]), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('executor containment still triggers on broken truth path alerts', () => {
|
||||||
|
assert.equal(shouldContainExecutorForAlerts([{
|
||||||
|
alert_code: 'near_intents_ingest_disconnected',
|
||||||
|
severity: 'critical',
|
||||||
|
}]), true);
|
||||||
|
assert.equal(shouldContainExecutorForAlerts([{
|
||||||
|
alert_code: 'near_intents_publish_stale',
|
||||||
|
severity: 'critical',
|
||||||
|
}]), true);
|
||||||
|
assert.equal(shouldContainExecutorForAlerts([{
|
||||||
|
alert_code: 'history_writer_stalled',
|
||||||
|
severity: 'critical',
|
||||||
|
}]), true);
|
||||||
|
});
|
||||||
Loading…
Add table
Reference in a new issue