Remove verifier salt refresh from executor hot path
All checks were successful
deploy / deploy (push) Successful in 58s
All checks were successful
deploy / deploy (push) Successful in 58s
Proof: quote-response execution now uses verifierSaltCache.getCachedFreshSalt() only, rejects verifier_salt_unavailable before signing or relay when no fresh cached salt exists, preserves stale command expiry before salt lookup, exposes salt cache and executor queue-delay state in health/sentinel/dashboard surfaces, and is covered by targeted regression tests plus full npm test and dashboard build. Assumptions: the existing verifier salt freshness policy remains safe for cached signing use; fixed in-code queue-delay warning thresholds are sufficient for operator alerts without new latency env vars; no strategy selection, pricing, edge, notional, inventory, arming, signer, or pair enablement behavior changed. Still fake: venue-native terminal fill ids and fee-complete realized PnL remain unavailable; executor concurrency remains single-consumer; live post-deploy evidence still needs collection after repo-workflow deployment.
This commit is contained in:
parent
8d2ca61fa1
commit
ddd3dfb9e2
16 changed files with 787 additions and 41 deletions
|
|
@ -14,6 +14,7 @@ import {
|
||||||
import { listDashboardServices } from '../core/operator-dashboard.mjs';
|
import { listDashboardServices } from '../core/operator-dashboard.mjs';
|
||||||
import {
|
import {
|
||||||
ageMs,
|
ageMs,
|
||||||
|
buildExecutorRuntimeAlerts,
|
||||||
buildMakerCompetitivenessRuntimeAlerts,
|
buildMakerCompetitivenessRuntimeAlerts,
|
||||||
buildRuntimeAlert,
|
buildRuntimeAlert,
|
||||||
createRuntimeHealthThresholds,
|
createRuntimeHealthThresholds,
|
||||||
|
|
@ -433,6 +434,12 @@ function buildDeterministicRuntimeAlerts({ servicesByName, now, previousRuntimeE
|
||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
alerts.push(...buildExecutorRuntimeAlerts({
|
||||||
|
executor,
|
||||||
|
thresholds,
|
||||||
|
now,
|
||||||
|
pair: config.activePair,
|
||||||
|
}));
|
||||||
|
|
||||||
const writer = servicesByName['history-writer'];
|
const writer = servicesByName['history-writer'];
|
||||||
const writerState = writer?.state || {};
|
const writerState = writer?.state || {};
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ import { createConsumer } from '../bus/kafka/consumer.mjs';
|
||||||
import { createProducer } from '../bus/kafka/producer.mjs';
|
import { createProducer } from '../bus/kafka/producer.mjs';
|
||||||
import { createArmedStateStore } from '../core/armed-state-store.mjs';
|
import { createArmedStateStore } from '../core/armed-state-store.mjs';
|
||||||
import { startControlApi } from '../core/control-api.mjs';
|
import { startControlApi } from '../core/control-api.mjs';
|
||||||
|
import { createExecutorQueueDelayTracker } from '../core/executor-queue-delay.mjs';
|
||||||
import { classifyExecuteCommandExpiry } from '../core/executor-command-expiry.mjs';
|
import { classifyExecuteCommandExpiry } from '../core/executor-command-expiry.mjs';
|
||||||
import { buildEventEnvelope, parseEventMessage } from '../core/event-envelope.mjs';
|
import { buildEventEnvelope, parseEventMessage } from '../core/event-envelope.mjs';
|
||||||
import { createExecutorStateStore } from '../core/executor-state-store.mjs';
|
import { createExecutorStateStore } from '../core/executor-state-store.mjs';
|
||||||
|
|
@ -107,6 +108,7 @@ const relayClient = await startSolverRelayWs({
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const stateStore = createExecutorStateStore({ stateDir: config.executorStateDir });
|
const stateStore = createExecutorStateStore({ stateDir: config.executorStateDir });
|
||||||
|
const executorQueueDelayTracker = createExecutorQueueDelayTracker();
|
||||||
const armedStateStore = createArmedStateStore({
|
const armedStateStore = createArmedStateStore({
|
||||||
stateDir: config.executorStateDir,
|
stateDir: config.executorStateDir,
|
||||||
fileName: 'trade-executor-control.json',
|
fileName: 'trade-executor-control.json',
|
||||||
|
|
@ -124,6 +126,9 @@ const state = {
|
||||||
last_error: null,
|
last_error: null,
|
||||||
in_flight_count: 0,
|
in_flight_count: 0,
|
||||||
submitted_count: 0,
|
submitted_count: 0,
|
||||||
|
salt_unavailable_reject_count: 0,
|
||||||
|
last_salt_unavailable_at: null,
|
||||||
|
last_salt_unavailable_state: null,
|
||||||
request_creation: {
|
request_creation: {
|
||||||
last_preflight: null,
|
last_preflight: null,
|
||||||
last_submission_result: null,
|
last_submission_result: null,
|
||||||
|
|
@ -189,6 +194,7 @@ await consumer.run({
|
||||||
async function handleCommand(event) {
|
async function handleCommand(event) {
|
||||||
const payload = event.payload;
|
const payload = event.payload;
|
||||||
const timing = startExecutorTiming(event);
|
const timing = startExecutorTiming(event);
|
||||||
|
recordExecutorQueueDelay(timing, payload);
|
||||||
state.last_command = payload;
|
state.last_command = payload;
|
||||||
|
|
||||||
const existing = stateStore.get(payload.command_id);
|
const existing = stateStore.get(payload.command_id);
|
||||||
|
|
@ -254,14 +260,32 @@ async function handleCommand(event) {
|
||||||
try {
|
try {
|
||||||
const saltStartMs = performance.now();
|
const saltStartMs = performance.now();
|
||||||
let currentSaltHex;
|
let currentSaltHex;
|
||||||
try {
|
const salt = verifierSaltCache.getCachedFreshSalt();
|
||||||
const salt = await verifierSaltCache.getFreshSalt();
|
recordExecutorTiming(timing, 'current_salt_ms', saltStartMs);
|
||||||
currentSaltHex = salt.currentSaltHex;
|
|
||||||
timing.current_salt_source = salt.source;
|
timing.current_salt_source = salt.source;
|
||||||
timing.current_salt_age_ms = roundTimingMs(salt.ageMs);
|
timing.current_salt_age_ms = roundTimingMs(salt.ageMs);
|
||||||
} finally {
|
timing.current_salt_unavailable_reason = salt.available ? null : salt.reason;
|
||||||
recordExecutorTiming(timing, 'current_salt_ms', saltStartMs);
|
timing.verifier_salt_cache_fresh = salt.state?.salt_fresh ?? null;
|
||||||
|
if (!salt.available) {
|
||||||
|
recordSaltUnavailable(salt);
|
||||||
|
stateStore.markFailed(payload.command_id, {
|
||||||
|
quote_id: payload.quote_id,
|
||||||
|
error: {
|
||||||
|
name: 'VerifierSaltUnavailable',
|
||||||
|
message: salt.reason || 'fresh verifier salt unavailable',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await publishResult(payload, withExecutorTiming({
|
||||||
|
status: 'rejected',
|
||||||
|
result_code: 'verifier_salt_unavailable',
|
||||||
|
failure_category: 'salt_unavailable',
|
||||||
|
note: 'fresh verifier salt unavailable; command was not signed or relayed',
|
||||||
|
verifier_salt_unavailable_reason: salt.reason || 'unavailable',
|
||||||
|
verifier_salt_cache: salt.state,
|
||||||
|
}, timing));
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
currentSaltHex = salt.currentSaltHex;
|
||||||
|
|
||||||
const signStartMs = performance.now();
|
const signStartMs = performance.now();
|
||||||
let submission;
|
let submission;
|
||||||
|
|
@ -336,6 +360,7 @@ function startExecutorTiming(event) {
|
||||||
received_at: receivedAt.toISOString(),
|
received_at: receivedAt.toISOString(),
|
||||||
started_at_ms: performance.now(),
|
started_at_ms: performance.now(),
|
||||||
maker_timing: makerTiming,
|
maker_timing: makerTiming,
|
||||||
|
command_to_executor_ms: makerTiming.command_to_executor_ms,
|
||||||
quote_age_at_executor_receipt_ms: makerTiming.quote_age_at_executor_receipt_ms,
|
quote_age_at_executor_receipt_ms: makerTiming.quote_age_at_executor_receipt_ms,
|
||||||
command_event_age_ms: Number.isFinite(commandEventAtMs)
|
command_event_age_ms: Number.isFinite(commandEventAtMs)
|
||||||
? roundTimingMs(receivedAt.getTime() - commandEventAtMs)
|
? roundTimingMs(receivedAt.getTime() - commandEventAtMs)
|
||||||
|
|
@ -374,11 +399,14 @@ function finishExecutorTiming(timing) {
|
||||||
executor_result_at: executorResultAt,
|
executor_result_at: executorResultAt,
|
||||||
relay_result_at: relayResultAt,
|
relay_result_at: relayResultAt,
|
||||||
command_event_age_ms: timing.command_event_age_ms,
|
command_event_age_ms: timing.command_event_age_ms,
|
||||||
|
command_to_executor_ms: timing.command_to_executor_ms ?? null,
|
||||||
quote_age_at_executor_receipt_ms: timing.quote_age_at_executor_receipt_ms ?? null,
|
quote_age_at_executor_receipt_ms: timing.quote_age_at_executor_receipt_ms ?? null,
|
||||||
quote_age_at_relay_result_ms: makerTiming.quote_age_at_relay_result_ms ?? null,
|
quote_age_at_relay_result_ms: makerTiming.quote_age_at_relay_result_ms ?? null,
|
||||||
current_salt_ms: timing.current_salt_ms ?? null,
|
current_salt_ms: timing.current_salt_ms ?? null,
|
||||||
current_salt_source: timing.current_salt_source ?? null,
|
current_salt_source: timing.current_salt_source ?? null,
|
||||||
current_salt_age_ms: timing.current_salt_age_ms ?? null,
|
current_salt_age_ms: timing.current_salt_age_ms ?? null,
|
||||||
|
current_salt_unavailable_reason: timing.current_salt_unavailable_reason ?? null,
|
||||||
|
verifier_salt_cache_fresh: timing.verifier_salt_cache_fresh ?? null,
|
||||||
sign_ms: timing.sign_ms ?? null,
|
sign_ms: timing.sign_ms ?? null,
|
||||||
relay_response_ms: timing.relay_response_ms ?? null,
|
relay_response_ms: timing.relay_response_ms ?? null,
|
||||||
executor_total_ms: roundTimingMs(performance.now() - timing.started_at_ms),
|
executor_total_ms: roundTimingMs(performance.now() - timing.started_at_ms),
|
||||||
|
|
@ -391,6 +419,22 @@ function roundTimingMs(value) {
|
||||||
return Number.isFinite(number) ? Math.round(number * 1000) / 1000 : null;
|
return Number.isFinite(number) ? Math.round(number * 1000) / 1000 : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function recordExecutorQueueDelay(timing, payload) {
|
||||||
|
executorQueueDelayTracker.record({
|
||||||
|
commandToExecutorMs: timing.command_to_executor_ms,
|
||||||
|
commandId: payload?.command_id || null,
|
||||||
|
quoteId: payload?.quote_id || null,
|
||||||
|
pair: payload?.pair || null,
|
||||||
|
at: timing.received_at,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function recordSaltUnavailable(salt) {
|
||||||
|
state.salt_unavailable_reject_count += 1;
|
||||||
|
state.last_salt_unavailable_at = new Date().toISOString();
|
||||||
|
state.last_salt_unavailable_state = salt.state || null;
|
||||||
|
}
|
||||||
|
|
||||||
async function publishResult(command, extraPayload) {
|
async function publishResult(command, extraPayload) {
|
||||||
const event = buildEventEnvelope({
|
const event = buildEventEnvelope({
|
||||||
source: 'trade-executor',
|
source: 'trade-executor',
|
||||||
|
|
@ -439,6 +483,7 @@ const controlApi = startControlApi({
|
||||||
signer_registered: signerRegistered,
|
signer_registered: signerRegistered,
|
||||||
relay: relayClient.getState(),
|
relay: relayClient.getState(),
|
||||||
verifier_salt_cache: verifierSaltCache.getState(),
|
verifier_salt_cache: verifierSaltCache.getState(),
|
||||||
|
executor_queue_delay: executorQueueDelayTracker.getState(),
|
||||||
trading_config: tradingConfigStore.getState(),
|
trading_config: tradingConfigStore.getState(),
|
||||||
...state,
|
...state,
|
||||||
durable_control_state: armedStateStore.getState(),
|
durable_control_state: armedStateStore.getState(),
|
||||||
|
|
@ -450,6 +495,8 @@ const controlApi = startControlApi({
|
||||||
getHealth() {
|
getHealth() {
|
||||||
const relay = relayClient.getState();
|
const relay = relayClient.getState();
|
||||||
const freshnessAgeMs = ageMs(relay.last_message_at);
|
const freshnessAgeMs = ageMs(relay.last_message_at);
|
||||||
|
const saltState = verifierSaltCache.getState();
|
||||||
|
const queueDelay = executorQueueDelayTracker.getState();
|
||||||
return {
|
return {
|
||||||
ok: relay.connected
|
ok: relay.connected
|
||||||
&& tradingConfigStore.getState().ok === true
|
&& tradingConfigStore.getState().ok === true
|
||||||
|
|
@ -459,6 +506,14 @@ const controlApi = startControlApi({
|
||||||
trading_config_block_reason: tradingConfigStore.getState().block_reason,
|
trading_config_block_reason: tradingConfigStore.getState().block_reason,
|
||||||
relay_last_message_at: relay.last_message_at,
|
relay_last_message_at: relay.last_message_at,
|
||||||
relay_freshness_age_ms: freshnessAgeMs,
|
relay_freshness_age_ms: freshnessAgeMs,
|
||||||
|
verifier_salt_cache: saltState,
|
||||||
|
verifier_salt_fresh: saltState.salt_fresh,
|
||||||
|
verifier_salt_age_ms: saltState.salt_age_ms,
|
||||||
|
verifier_salt_refresh_in_flight: saltState.refresh_in_flight,
|
||||||
|
verifier_salt_last_refresh_error: saltState.last_refresh_error,
|
||||||
|
executor_queue_delay: queueDelay,
|
||||||
|
executor_queue_delay_status: queueDelay.status,
|
||||||
|
executor_queue_delay_warning_reason: queueDelay.warning_reason,
|
||||||
paused: state.paused,
|
paused: state.paused,
|
||||||
armed: state.armed,
|
armed: state.armed,
|
||||||
reason:
|
reason:
|
||||||
|
|
|
||||||
127
src/core/executor-queue-delay.mjs
Normal file
127
src/core/executor-queue-delay.mjs
Normal file
|
|
@ -0,0 +1,127 @@
|
||||||
|
export const DEFAULT_EXECUTOR_QUEUE_DELAY_WARNING_MS = 1_000;
|
||||||
|
export const DEFAULT_EXECUTOR_QUEUE_DELAY_CRITICAL_MS = 5_000;
|
||||||
|
export const DEFAULT_EXECUTOR_QUEUE_DELAY_SAMPLE_LIMIT = 50;
|
||||||
|
|
||||||
|
export function createExecutorQueueDelayTracker({
|
||||||
|
sampleLimit = DEFAULT_EXECUTOR_QUEUE_DELAY_SAMPLE_LIMIT,
|
||||||
|
warningMs = DEFAULT_EXECUTOR_QUEUE_DELAY_WARNING_MS,
|
||||||
|
criticalMs = DEFAULT_EXECUTOR_QUEUE_DELAY_CRITICAL_MS,
|
||||||
|
now = () => Date.now(),
|
||||||
|
} = {}) {
|
||||||
|
let samples = [];
|
||||||
|
|
||||||
|
function record({
|
||||||
|
commandToExecutorMs,
|
||||||
|
commandId = null,
|
||||||
|
quoteId = null,
|
||||||
|
pair = null,
|
||||||
|
at = now(),
|
||||||
|
} = {}) {
|
||||||
|
const value = roundTimingMs(commandToExecutorMs);
|
||||||
|
if (value == null) return getState();
|
||||||
|
|
||||||
|
samples.push({
|
||||||
|
at: toIsoTimestamp(at),
|
||||||
|
command_id: commandId,
|
||||||
|
quote_id: quoteId,
|
||||||
|
pair,
|
||||||
|
command_to_executor_ms: value,
|
||||||
|
});
|
||||||
|
samples = samples.slice(-positiveInteger(sampleLimit, DEFAULT_EXECUTOR_QUEUE_DELAY_SAMPLE_LIMIT));
|
||||||
|
return getState();
|
||||||
|
}
|
||||||
|
|
||||||
|
function getState() {
|
||||||
|
return summarizeExecutorQueueDelaySamples(samples, {
|
||||||
|
warningMs,
|
||||||
|
criticalMs,
|
||||||
|
sampleLimit,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
record,
|
||||||
|
getState,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function summarizeExecutorQueueDelaySamples(samples = [], {
|
||||||
|
warningMs = DEFAULT_EXECUTOR_QUEUE_DELAY_WARNING_MS,
|
||||||
|
criticalMs = DEFAULT_EXECUTOR_QUEUE_DELAY_CRITICAL_MS,
|
||||||
|
sampleLimit = DEFAULT_EXECUTOR_QUEUE_DELAY_SAMPLE_LIMIT,
|
||||||
|
} = {}) {
|
||||||
|
const normalizedSamples = (samples || [])
|
||||||
|
.map((sample) => ({
|
||||||
|
at: toIsoTimestamp(sample.at),
|
||||||
|
command_id: sample.command_id || null,
|
||||||
|
quote_id: sample.quote_id || null,
|
||||||
|
pair: sample.pair || null,
|
||||||
|
command_to_executor_ms: roundTimingMs(sample.command_to_executor_ms),
|
||||||
|
}))
|
||||||
|
.filter((sample) => sample.command_to_executor_ms != null)
|
||||||
|
.slice(-positiveInteger(sampleLimit, DEFAULT_EXECUTOR_QUEUE_DELAY_SAMPLE_LIMIT));
|
||||||
|
const values = normalizedSamples
|
||||||
|
.map((sample) => sample.command_to_executor_ms)
|
||||||
|
.sort((left, right) => left - right);
|
||||||
|
const latest = normalizedSamples.at(-1) || null;
|
||||||
|
const maxMs = values.length ? values.at(-1) : null;
|
||||||
|
const status = maxMs == null
|
||||||
|
? 'unknown'
|
||||||
|
: maxMs >= criticalMs
|
||||||
|
? 'critical'
|
||||||
|
: maxMs >= warningMs
|
||||||
|
? 'warning'
|
||||||
|
: 'healthy';
|
||||||
|
const thresholdMs = status === 'critical' ? criticalMs : warningMs;
|
||||||
|
|
||||||
|
return {
|
||||||
|
status,
|
||||||
|
warning: status === 'warning' || status === 'critical',
|
||||||
|
critical: status === 'critical',
|
||||||
|
warning_reason: status === 'warning' || status === 'critical'
|
||||||
|
? `recent command_to_executor_ms max ${maxMs}ms exceeds ${thresholdMs}ms`
|
||||||
|
: null,
|
||||||
|
warning_after_ms: warningMs,
|
||||||
|
critical_after_ms: criticalMs,
|
||||||
|
sample_count: normalizedSamples.length,
|
||||||
|
latest_ms: latest?.command_to_executor_ms ?? null,
|
||||||
|
latest_at: latest?.at || null,
|
||||||
|
latest_command_id: latest?.command_id || null,
|
||||||
|
latest_quote_id: latest?.quote_id || null,
|
||||||
|
latest_pair: latest?.pair || null,
|
||||||
|
max_ms: maxMs,
|
||||||
|
p50_ms: percentile(values, 0.5),
|
||||||
|
p90_ms: percentile(values, 0.9),
|
||||||
|
p99_ms: percentile(values, 0.99),
|
||||||
|
recent_samples: normalizedSamples.slice(-10),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function percentile(sortedValues, ratio) {
|
||||||
|
if (!sortedValues.length) return null;
|
||||||
|
const index = Math.min(
|
||||||
|
sortedValues.length - 1,
|
||||||
|
Math.max(0, Math.ceil(sortedValues.length * ratio) - 1),
|
||||||
|
);
|
||||||
|
return sortedValues[index];
|
||||||
|
}
|
||||||
|
|
||||||
|
function roundTimingMs(value) {
|
||||||
|
const number = Number(value);
|
||||||
|
return Number.isFinite(number) ? Math.round(number * 1000) / 1000 : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function positiveInteger(value, fallback) {
|
||||||
|
const parsed = Number(value);
|
||||||
|
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function toIsoTimestamp(value) {
|
||||||
|
if (!value) return null;
|
||||||
|
if (typeof value === 'string') {
|
||||||
|
const parsed = Date.parse(value);
|
||||||
|
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
|
||||||
|
}
|
||||||
|
const date = new Date(value);
|
||||||
|
return Number.isNaN(date.getTime()) ? null : date.toISOString();
|
||||||
|
}
|
||||||
|
|
@ -2093,9 +2093,13 @@ function buildServiceSummary(service, state) {
|
||||||
return {
|
return {
|
||||||
in_flight_count: state.in_flight_count ?? 0,
|
in_flight_count: state.in_flight_count ?? 0,
|
||||||
submitted_count: state.submitted_count ?? state.completed_count ?? 0,
|
submitted_count: state.submitted_count ?? state.completed_count ?? 0,
|
||||||
|
salt_unavailable_reject_count: state.salt_unavailable_reject_count ?? 0,
|
||||||
|
last_salt_unavailable_at: state.last_salt_unavailable_at || null,
|
||||||
signer_registered: state.signer_registered ?? null,
|
signer_registered: state.signer_registered ?? null,
|
||||||
relay_connected: state.relay?.connected ?? null,
|
relay_connected: state.relay?.connected ?? null,
|
||||||
relay_last_message_at: state.relay?.last_message_at || null,
|
relay_last_message_at: state.relay?.last_message_at || null,
|
||||||
|
verifier_salt_cache: state.verifier_salt_cache || null,
|
||||||
|
executor_queue_delay: state.executor_queue_delay || null,
|
||||||
};
|
};
|
||||||
case 'operator-dashboard':
|
case 'operator-dashboard':
|
||||||
return {
|
return {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
export const SERVICE_HEALTH_LEVELS = ['healthy', 'warning', 'critical', 'offline', 'paused'];
|
export const SERVICE_HEALTH_LEVELS = ['healthy', 'warning', 'critical', 'offline', 'paused'];
|
||||||
|
export const DEFAULT_EXECUTOR_QUEUE_DELAY_WARNING_MS = 1_000;
|
||||||
|
export const DEFAULT_EXECUTOR_QUEUE_DELAY_CRITICAL_MS = 5_000;
|
||||||
|
|
||||||
const HEALTH_RANK = {
|
const HEALTH_RANK = {
|
||||||
healthy: 0,
|
healthy: 0,
|
||||||
|
|
@ -27,6 +29,12 @@ export function createRuntimeHealthThresholds(config = {}) {
|
||||||
anomalyQuoteRateCollapseRatio: Number(config.opsSentinelAnomalyQuoteRateCollapseRatio || 0.25),
|
anomalyQuoteRateCollapseRatio: Number(config.opsSentinelAnomalyQuoteRateCollapseRatio || 0.25),
|
||||||
anomalyReconnectSpikeMultiplier: Number(config.opsSentinelAnomalyReconnectSpikeMultiplier || 2),
|
anomalyReconnectSpikeMultiplier: Number(config.opsSentinelAnomalyReconnectSpikeMultiplier || 2),
|
||||||
containmentCooldownMs: Number(config.opsSentinelContainmentCooldownMs || 60_000),
|
containmentCooldownMs: Number(config.opsSentinelContainmentCooldownMs || 60_000),
|
||||||
|
executorQueueDelayWarningMs: Number(
|
||||||
|
config.executorQueueDelayWarningMs || DEFAULT_EXECUTOR_QUEUE_DELAY_WARNING_MS,
|
||||||
|
),
|
||||||
|
executorQueueDelayCriticalMs: Number(
|
||||||
|
config.executorQueueDelayCriticalMs || DEFAULT_EXECUTOR_QUEUE_DELAY_CRITICAL_MS,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -129,6 +137,31 @@ export function deriveServiceHealth({
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (service === 'trade-executor') {
|
||||||
|
const saltState = state.verifier_salt_cache || health.verifier_salt_cache || null;
|
||||||
|
const queueDelay = state.executor_queue_delay || health.executor_queue_delay || null;
|
||||||
|
|
||||||
|
if (saltState?.configured && saltState.salt_fresh !== true) {
|
||||||
|
status = escalateHealth(status, 'warning');
|
||||||
|
if (label === 'healthy') label = 'salt unavailable';
|
||||||
|
reasons.push('verifier salt cache is stale or unavailable');
|
||||||
|
}
|
||||||
|
if (saltState?.last_refresh_error) {
|
||||||
|
status = escalateHealth(status, 'warning');
|
||||||
|
if (label === 'healthy') label = 'salt refresh failed';
|
||||||
|
reasons.push('verifier salt refresh failed');
|
||||||
|
}
|
||||||
|
if (queueDelay?.status === 'critical') {
|
||||||
|
status = escalateHealth(status, 'critical');
|
||||||
|
label = 'queue delayed';
|
||||||
|
reasons.push(queueDelay.warning_reason || 'executor queue delay is critical');
|
||||||
|
} else if (queueDelay?.status === 'warning') {
|
||||||
|
status = escalateHealth(status, 'warning');
|
||||||
|
if (label === 'healthy') label = 'queue delayed';
|
||||||
|
reasons.push(queueDelay.warning_reason || 'executor queue delay is high');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (service === 'history-writer') {
|
if (service === 'history-writer') {
|
||||||
if (state.database_connectivity === false) {
|
if (state.database_connectivity === false) {
|
||||||
status = escalateHealth(status, 'critical');
|
status = escalateHealth(status, 'critical');
|
||||||
|
|
@ -379,6 +412,90 @@ export function buildMakerCompetitivenessRuntimeAlerts({
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function buildExecutorRuntimeAlerts({
|
||||||
|
executor,
|
||||||
|
thresholds = createRuntimeHealthThresholds(),
|
||||||
|
now = new Date().toISOString(),
|
||||||
|
pair = null,
|
||||||
|
} = {}) {
|
||||||
|
void now;
|
||||||
|
const alerts = [];
|
||||||
|
const state = executor?.state || {};
|
||||||
|
const health = executor?.health || {};
|
||||||
|
const saltState = state.verifier_salt_cache || health.verifier_salt_cache || null;
|
||||||
|
const queueDelay = state.executor_queue_delay || health.executor_queue_delay || null;
|
||||||
|
|
||||||
|
if (saltState?.configured && saltState.salt_fresh !== true) {
|
||||||
|
alerts.push(buildRuntimeAlert({
|
||||||
|
alert_code: 'trade_executor_verifier_salt_unavailable',
|
||||||
|
severity: 'warning',
|
||||||
|
reason: saltState.has_cached_salt
|
||||||
|
? 'trade-executor verifier salt cache is stale'
|
||||||
|
: 'trade-executor verifier salt cache is unavailable',
|
||||||
|
service_scope: 'trade-executor',
|
||||||
|
pair,
|
||||||
|
details: {
|
||||||
|
has_cached_salt: saltState.has_cached_salt ?? null,
|
||||||
|
salt_fresh: saltState.salt_fresh ?? saltState.fresh ?? null,
|
||||||
|
salt_age_ms: saltState.salt_age_ms ?? saltState.cached_age_ms ?? null,
|
||||||
|
max_age_ms: saltState.max_age_ms ?? null,
|
||||||
|
refresh_in_flight: saltState.refresh_in_flight ?? null,
|
||||||
|
last_refresh_started_at: saltState.last_refresh_started_at || null,
|
||||||
|
last_refresh_completed_at: saltState.last_refresh_completed_at || null,
|
||||||
|
last_refresh_error: saltState.last_refresh_error || null,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (saltState?.last_refresh_error) {
|
||||||
|
alerts.push(buildRuntimeAlert({
|
||||||
|
alert_code: 'trade_executor_verifier_salt_refresh_failed',
|
||||||
|
severity: 'warning',
|
||||||
|
reason: 'trade-executor verifier salt refresh failed',
|
||||||
|
service_scope: 'trade-executor',
|
||||||
|
pair,
|
||||||
|
details: {
|
||||||
|
last_refresh_error: saltState.last_refresh_error,
|
||||||
|
last_refresh_started_at: saltState.last_refresh_started_at || null,
|
||||||
|
last_refresh_completed_at: saltState.last_refresh_completed_at || null,
|
||||||
|
last_refresh_duration_ms: saltState.last_refresh_duration_ms ?? null,
|
||||||
|
refresh_in_flight: saltState.refresh_in_flight ?? null,
|
||||||
|
has_cached_salt: saltState.has_cached_salt ?? null,
|
||||||
|
salt_fresh: saltState.salt_fresh ?? saltState.fresh ?? null,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
const queueMaxMs = Number(queueDelay?.max_ms);
|
||||||
|
if (Number.isFinite(queueMaxMs) && queueMaxMs >= thresholds.executorQueueDelayWarningMs) {
|
||||||
|
const critical = queueMaxMs >= thresholds.executorQueueDelayCriticalMs;
|
||||||
|
alerts.push(buildRuntimeAlert({
|
||||||
|
alert_code: 'trade_executor_command_queue_delay_high',
|
||||||
|
severity: critical ? 'critical' : 'warning',
|
||||||
|
reason: `trade-executor command queue delay ${queueMaxMs}ms exceeds ${
|
||||||
|
critical ? thresholds.executorQueueDelayCriticalMs : thresholds.executorQueueDelayWarningMs
|
||||||
|
}ms`,
|
||||||
|
service_scope: 'trade-executor',
|
||||||
|
pair,
|
||||||
|
details: {
|
||||||
|
status: queueDelay.status || (critical ? 'critical' : 'warning'),
|
||||||
|
latest_ms: queueDelay.latest_ms ?? null,
|
||||||
|
max_ms: queueDelay.max_ms ?? null,
|
||||||
|
p50_ms: queueDelay.p50_ms ?? null,
|
||||||
|
p90_ms: queueDelay.p90_ms ?? null,
|
||||||
|
p99_ms: queueDelay.p99_ms ?? null,
|
||||||
|
sample_count: queueDelay.sample_count ?? null,
|
||||||
|
warning_after_ms: thresholds.executorQueueDelayWarningMs,
|
||||||
|
critical_after_ms: thresholds.executorQueueDelayCriticalMs,
|
||||||
|
latest_command_id: queueDelay.latest_command_id || null,
|
||||||
|
latest_quote_id: queueDelay.latest_quote_id || null,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
return alerts;
|
||||||
|
}
|
||||||
|
|
||||||
export function shouldRaiseIngestPublishStale({
|
export function shouldRaiseIngestPublishStale({
|
||||||
lastMatchingQuoteAt = null,
|
lastMatchingQuoteAt = null,
|
||||||
lastPublishedAt = null,
|
lastPublishedAt = null,
|
||||||
|
|
|
||||||
|
|
@ -109,9 +109,14 @@ export function summarizeServiceState(service, state) {
|
||||||
'armed',
|
'armed',
|
||||||
'last_error',
|
'last_error',
|
||||||
'relay',
|
'relay',
|
||||||
|
'verifier_salt_cache',
|
||||||
|
'executor_queue_delay',
|
||||||
'last_result',
|
'last_result',
|
||||||
'last_quote_status',
|
'last_quote_status',
|
||||||
'submitted_count',
|
'submitted_count',
|
||||||
|
'salt_unavailable_reject_count',
|
||||||
|
'last_salt_unavailable_at',
|
||||||
|
'last_salt_unavailable_state',
|
||||||
'failed_count',
|
'failed_count',
|
||||||
'blocked_count',
|
'blocked_count',
|
||||||
]);
|
]);
|
||||||
|
|
|
||||||
|
|
@ -18,30 +18,47 @@ export function createVerifierSaltCache({
|
||||||
let cached = null;
|
let cached = null;
|
||||||
let refreshInFlight = null;
|
let refreshInFlight = null;
|
||||||
let timer = null;
|
let timer = null;
|
||||||
|
let lastRefreshStartedAtMs = null;
|
||||||
|
let lastRefreshStartedAt = null;
|
||||||
|
let lastRefreshCompletedAt = null;
|
||||||
|
let lastRefreshDurationMs = null;
|
||||||
|
let lastRefreshReason = null;
|
||||||
|
let refreshInFlightReason = null;
|
||||||
let lastRefreshError = null;
|
let lastRefreshError = null;
|
||||||
let lastRefreshErrorLoggedAtMs = 0;
|
let lastRefreshErrorLoggedAtMs = 0;
|
||||||
|
|
||||||
async function refresh({ reason = 'manual' } = {}) {
|
async function refreshNow({ reason = 'manual' } = {}) {
|
||||||
if (refreshInFlight) return refreshInFlight;
|
if (refreshInFlight) return refreshInFlight;
|
||||||
|
|
||||||
|
const startedAtMs = now();
|
||||||
|
lastRefreshStartedAtMs = startedAtMs;
|
||||||
|
lastRefreshStartedAt = toIsoTimestamp(startedAtMs);
|
||||||
|
lastRefreshReason = reason;
|
||||||
|
refreshInFlightReason = reason;
|
||||||
|
|
||||||
refreshInFlight = Promise.resolve()
|
refreshInFlight = Promise.resolve()
|
||||||
.then(() => loadSalt())
|
.then(() => loadSalt())
|
||||||
.then((salt) => {
|
.then((salt) => {
|
||||||
|
const refreshedAtMs = now();
|
||||||
cached = {
|
cached = {
|
||||||
currentSaltHex: normalizeSalt(salt),
|
currentSaltHex: normalizeSalt(salt),
|
||||||
refreshedAtMs: now(),
|
refreshedAtMs,
|
||||||
|
refreshedAt: toIsoTimestamp(refreshedAtMs),
|
||||||
refreshedReason: reason,
|
refreshedReason: reason,
|
||||||
};
|
};
|
||||||
lastRefreshError = null;
|
lastRefreshError = null;
|
||||||
|
recordRefreshCompleted();
|
||||||
return buildSaltResult(cached, 'refresh');
|
return buildSaltResult(cached, 'refresh');
|
||||||
})
|
})
|
||||||
.catch((error) => {
|
.catch((error) => {
|
||||||
logRefreshError(error, reason);
|
logRefreshError(error, reason);
|
||||||
lastRefreshError = serializeSaltError(error);
|
lastRefreshError = serializeSaltError(error);
|
||||||
|
recordRefreshCompleted();
|
||||||
throw error;
|
throw error;
|
||||||
})
|
})
|
||||||
.finally(() => {
|
.finally(() => {
|
||||||
refreshInFlight = null;
|
refreshInFlight = null;
|
||||||
|
refreshInFlightReason = null;
|
||||||
});
|
});
|
||||||
|
|
||||||
return refreshInFlight;
|
return refreshInFlight;
|
||||||
|
|
@ -50,18 +67,55 @@ export function createVerifierSaltCache({
|
||||||
async function getFreshSalt() {
|
async function getFreshSalt() {
|
||||||
if (isFresh(cached)) return buildSaltResult(cached, 'cache');
|
if (isFresh(cached)) return buildSaltResult(cached, 'cache');
|
||||||
|
|
||||||
await refresh({ reason: cached ? 'stale_cache' : 'empty_cache' });
|
await refreshNow({ reason: cached ? 'stale_cache' : 'empty_cache' });
|
||||||
if (!isFresh(cached)) {
|
if (!isFresh(cached)) {
|
||||||
throw new Error('verifier salt cache refresh did not produce a fresh salt');
|
throw new Error('verifier salt cache refresh did not produce a fresh salt');
|
||||||
}
|
}
|
||||||
return buildSaltResult(cached, 'refresh');
|
return buildSaltResult(cached, 'refresh');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getCachedFreshSalt(options = {}) {
|
||||||
|
const readNowMs = options.now == null ? now() : timestampMs(options.now);
|
||||||
|
const readMaxAgeMs = positiveNumber(options.maxAgeMs, maxAgeMs);
|
||||||
|
const state = getState({ now: readNowMs, maxAgeMs: readMaxAgeMs });
|
||||||
|
|
||||||
|
if (!cached) {
|
||||||
|
return {
|
||||||
|
available: false,
|
||||||
|
source: 'unavailable',
|
||||||
|
reason: 'empty_cache',
|
||||||
|
ageMs: null,
|
||||||
|
state,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const ageMs = readNowMs - cached.refreshedAtMs;
|
||||||
|
if (!isFresh(cached, { nowMs: readNowMs, maxAgeMs: readMaxAgeMs })) {
|
||||||
|
return {
|
||||||
|
available: false,
|
||||||
|
source: 'unavailable',
|
||||||
|
reason: ageMs < 0 ? 'clock_skew' : 'stale_cache',
|
||||||
|
ageMs: Number.isFinite(ageMs) ? Math.max(0, ageMs) : null,
|
||||||
|
state,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
available: true,
|
||||||
|
currentSaltHex: cached.currentSaltHex,
|
||||||
|
source: 'cache',
|
||||||
|
ageMs: Math.max(0, ageMs),
|
||||||
|
refreshedAt: cached.refreshedAt,
|
||||||
|
refreshedReason: cached.refreshedReason,
|
||||||
|
state,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function start() {
|
function start() {
|
||||||
if (timer) return;
|
if (timer) return;
|
||||||
void refresh({ reason: 'startup' }).catch(() => {});
|
void refreshNow({ reason: 'startup' }).catch(() => {});
|
||||||
timer = setIntervalFn(() => {
|
timer = setIntervalFn(() => {
|
||||||
void refresh({ reason: 'prefetch' }).catch(() => {});
|
void refreshNow({ reason: 'prefetch' }).catch(() => {});
|
||||||
}, refreshIntervalMs);
|
}, refreshIntervalMs);
|
||||||
timer?.unref?.();
|
timer?.unref?.();
|
||||||
}
|
}
|
||||||
|
|
@ -72,25 +126,37 @@ export function createVerifierSaltCache({
|
||||||
timer = null;
|
timer = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getState() {
|
function getState(options = {}) {
|
||||||
const ageMs = cached ? now() - cached.refreshedAtMs : null;
|
const stateNowMs = options.now == null ? now() : timestampMs(options.now);
|
||||||
|
const stateMaxAgeMs = positiveNumber(options.maxAgeMs, maxAgeMs);
|
||||||
|
const ageMs = cached ? stateNowMs - cached.refreshedAtMs : null;
|
||||||
|
const fresh = isFresh(cached, { nowMs: stateNowMs, maxAgeMs: stateMaxAgeMs });
|
||||||
|
const roundedAgeMs = Number.isFinite(ageMs) ? Math.max(0, Math.round(ageMs)) : null;
|
||||||
return {
|
return {
|
||||||
configured: true,
|
configured: true,
|
||||||
has_cached_salt: Boolean(cached),
|
has_cached_salt: Boolean(cached),
|
||||||
fresh: isFresh(cached),
|
fresh,
|
||||||
max_age_ms: maxAgeMs,
|
salt_fresh: fresh,
|
||||||
|
salt_age_ms: roundedAgeMs,
|
||||||
|
max_age_ms: stateMaxAgeMs,
|
||||||
refresh_interval_ms: refreshIntervalMs,
|
refresh_interval_ms: refreshIntervalMs,
|
||||||
cached_age_ms: ageMs == null ? null : Math.max(0, Math.round(ageMs)),
|
cached_age_ms: roundedAgeMs,
|
||||||
|
cached_refreshed_at: cached?.refreshedAt || null,
|
||||||
refreshed_reason: cached?.refreshedReason || null,
|
refreshed_reason: cached?.refreshedReason || null,
|
||||||
refresh_in_flight: Boolean(refreshInFlight),
|
refresh_in_flight: Boolean(refreshInFlight),
|
||||||
|
refresh_in_flight_reason: refreshInFlightReason,
|
||||||
|
last_refresh_started_at: lastRefreshStartedAt,
|
||||||
|
last_refresh_completed_at: lastRefreshCompletedAt,
|
||||||
|
last_refresh_duration_ms: lastRefreshDurationMs,
|
||||||
|
last_refresh_reason: lastRefreshReason,
|
||||||
last_refresh_error: lastRefreshError,
|
last_refresh_error: lastRefreshError,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function isFresh(entry) {
|
function isFresh(entry, { nowMs = now(), maxAgeMs: readMaxAgeMs = maxAgeMs } = {}) {
|
||||||
if (!entry) return false;
|
if (!entry) return false;
|
||||||
const ageMs = now() - entry.refreshedAtMs;
|
const ageMs = nowMs - entry.refreshedAtMs;
|
||||||
return ageMs >= 0 && ageMs <= maxAgeMs;
|
return Number.isFinite(ageMs) && ageMs >= 0 && ageMs <= readMaxAgeMs;
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildSaltResult(entry, source) {
|
function buildSaltResult(entry, source) {
|
||||||
|
|
@ -102,6 +168,14 @@ export function createVerifierSaltCache({
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function recordRefreshCompleted() {
|
||||||
|
const completedAtMs = now();
|
||||||
|
lastRefreshCompletedAt = toIsoTimestamp(completedAtMs);
|
||||||
|
lastRefreshDurationMs = lastRefreshStartedAtMs == null
|
||||||
|
? null
|
||||||
|
: Math.max(0, Math.round((completedAtMs - lastRefreshStartedAtMs) * 1000) / 1000);
|
||||||
|
}
|
||||||
|
|
||||||
function logRefreshError(error, reason) {
|
function logRefreshError(error, reason) {
|
||||||
if (!logger?.warn) return;
|
if (!logger?.warn) return;
|
||||||
const serialized = serializeSaltError(error);
|
const serialized = serializeSaltError(error);
|
||||||
|
|
@ -119,9 +193,11 @@ export function createVerifierSaltCache({
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
getCachedFreshSalt,
|
||||||
getFreshSalt,
|
getFreshSalt,
|
||||||
getState,
|
getState,
|
||||||
refresh,
|
refresh: refreshNow,
|
||||||
|
refreshNow,
|
||||||
start,
|
start,
|
||||||
stop,
|
stop,
|
||||||
};
|
};
|
||||||
|
|
@ -141,3 +217,20 @@ function serializeSaltError(error) {
|
||||||
message: error?.message || String(error),
|
message: error?.message || String(error),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function positiveNumber(value, fallback) {
|
||||||
|
const parsed = Number(value);
|
||||||
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function timestampMs(value) {
|
||||||
|
if (value == null) return Number.NaN;
|
||||||
|
if (typeof value === 'number') return value;
|
||||||
|
if (value instanceof Date) return value.getTime();
|
||||||
|
return Date.parse(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function toIsoTimestamp(value) {
|
||||||
|
const parsed = timestampMs(value);
|
||||||
|
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,9 @@ function useNow(intervalMs = 1000) {
|
||||||
export default function ServiceCard({ service }) {
|
export default function ServiceCard({ service }) {
|
||||||
const healthLabel = service.health_label || service.health_status || (service.reachable ? 'online' : 'offline');
|
const healthLabel = service.health_label || service.health_status || (service.reachable ? 'online' : 'offline');
|
||||||
const now = useNow();
|
const now = useNow();
|
||||||
|
const summary = service.summary || {};
|
||||||
|
const saltCache = summary.verifier_salt_cache || null;
|
||||||
|
const queueDelay = summary.executor_queue_delay || null;
|
||||||
const freshnessAge = service.freshness_at
|
const freshnessAge = service.freshness_at
|
||||||
? formatAgeFromTimestamp(service.freshness_at, now)
|
? formatAgeFromTimestamp(service.freshness_at, now)
|
||||||
: formatAge(service.freshness_age_ms);
|
: formatAge(service.freshness_age_ms);
|
||||||
|
|
@ -42,9 +45,43 @@ export default function ServiceCard({ service }) {
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
|
{saltCache ? (
|
||||||
|
<>
|
||||||
|
<div>{`Verifier salt ${formatSaltFreshness(saltCache)}`}</div>
|
||||||
|
<div>{`Salt age ${formatAge(saltCache.salt_age_ms ?? saltCache.cached_age_ms)}`}</div>
|
||||||
|
<div>{`Salt refresh ${saltCache.refresh_in_flight ? 'in flight' : 'idle'}`}</div>
|
||||||
|
<div>{`Salt refreshed ${formatTimestamp(saltCache.last_refresh_completed_at || saltCache.cached_refreshed_at)}`}</div>
|
||||||
|
<div>{`Salt refresh duration ${formatAge(saltCache.last_refresh_duration_ms)}`}</div>
|
||||||
|
{saltCache.last_refresh_error ? (
|
||||||
|
<div>{`Salt error ${formatError(saltCache.last_refresh_error)}`}</div>
|
||||||
|
) : null}
|
||||||
|
<div>{`Salt rejects ${summary.salt_unavailable_reject_count ?? 0}`}</div>
|
||||||
|
</>
|
||||||
|
) : null}
|
||||||
|
{queueDelay ? (
|
||||||
|
<>
|
||||||
|
<div>{`Queue delay ${queueDelay.status || 'unknown'}`}</div>
|
||||||
|
<div>{`Queue latest ${formatAge(queueDelay.latest_ms)}`}</div>
|
||||||
|
<div>{`Queue p90 ${formatAge(queueDelay.p90_ms)}`}</div>
|
||||||
|
<div>{`Queue p99 ${formatAge(queueDelay.p99_ms)}`}</div>
|
||||||
|
<div>{`Queue max ${formatAge(queueDelay.max_ms)}`}</div>
|
||||||
|
{queueDelay.warning_reason ? <div>{queueDelay.warning_reason}</div> : null}
|
||||||
|
</>
|
||||||
|
) : null}
|
||||||
<div className="mono">{service.base_url}</div>
|
<div className="mono">{service.base_url}</div>
|
||||||
{service.last_error ? <div>{JSON.stringify(service.last_error)}</div> : null}
|
{service.last_error ? <div>{JSON.stringify(service.last_error)}</div> : null}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatSaltFreshness(saltCache) {
|
||||||
|
if (saltCache.salt_fresh === true || saltCache.fresh === true) return 'fresh';
|
||||||
|
if (saltCache.has_cached_salt) return 'stale';
|
||||||
|
return 'unavailable';
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatError(error) {
|
||||||
|
if (!error) return 'Unavailable';
|
||||||
|
return error.message || JSON.stringify(error);
|
||||||
|
}
|
||||||
|
|
|
||||||
36
test/executor-queue-delay.test.mjs
Normal file
36
test/executor-queue-delay.test.mjs
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
import { createExecutorQueueDelayTracker } from '../src/core/executor-queue-delay.mjs';
|
||||||
|
|
||||||
|
test('executor queue delay tracker records recent percentiles and warning state', () => {
|
||||||
|
let nowMs = Date.parse('2026-06-12T10:00:00.000Z');
|
||||||
|
const tracker = createExecutorQueueDelayTracker({
|
||||||
|
sampleLimit: 5,
|
||||||
|
warningMs: 1_000,
|
||||||
|
criticalMs: 5_000,
|
||||||
|
now: () => nowMs,
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const value of [10, 25, 100, 1_250, 7_500]) {
|
||||||
|
tracker.record({
|
||||||
|
commandToExecutorMs: value,
|
||||||
|
commandId: `cmd-${value}`,
|
||||||
|
quoteId: `quote-${value}`,
|
||||||
|
pair: 'nbtc->eure',
|
||||||
|
});
|
||||||
|
nowMs += 1_000;
|
||||||
|
}
|
||||||
|
|
||||||
|
const state = tracker.getState();
|
||||||
|
assert.equal(state.status, 'critical');
|
||||||
|
assert.equal(state.warning, true);
|
||||||
|
assert.equal(state.critical, true);
|
||||||
|
assert.equal(state.sample_count, 5);
|
||||||
|
assert.equal(state.latest_ms, 7_500);
|
||||||
|
assert.equal(state.max_ms, 7_500);
|
||||||
|
assert.equal(state.p50_ms, 100);
|
||||||
|
assert.equal(state.p90_ms, 7_500);
|
||||||
|
assert.equal(state.p99_ms, 7_500);
|
||||||
|
assert.match(state.warning_reason, /command_to_executor_ms/);
|
||||||
|
});
|
||||||
|
|
@ -98,6 +98,19 @@ test('dashboard freshness surfaces show age and exact timestamp evidence', () =>
|
||||||
assert.match(stylesSource, /@keyframes quote-row-flash-cell/);
|
assert.match(stylesSource, /@keyframes quote-row-flash-cell/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('service cards expose executor verifier salt and queue delay state', () => {
|
||||||
|
assert.match(serviceCardSource, /verifier_salt_cache/);
|
||||||
|
assert.match(serviceCardSource, /executor_queue_delay/);
|
||||||
|
assert.match(serviceCardSource, /Verifier salt/);
|
||||||
|
assert.match(serviceCardSource, /Salt age/);
|
||||||
|
assert.match(serviceCardSource, /Salt refresh/);
|
||||||
|
assert.match(serviceCardSource, /Salt error/);
|
||||||
|
assert.match(serviceCardSource, /Salt rejects/);
|
||||||
|
assert.match(serviceCardSource, /Queue delay/);
|
||||||
|
assert.match(serviceCardSource, /Queue p90/);
|
||||||
|
assert.match(serviceCardSource, /Queue p99/);
|
||||||
|
});
|
||||||
|
|
||||||
test('mobile status bar uses normal document flow instead of sticky viewport positioning', () => {
|
test('mobile status bar uses normal document flow instead of sticky viewport positioning', () => {
|
||||||
assert.match(
|
assert.match(
|
||||||
stylesSource,
|
stylesSource,
|
||||||
|
|
|
||||||
|
|
@ -2086,6 +2086,65 @@ test('dashboard does not pause relay services for unrelated destination-chain in
|
||||||
assert.equal(services['trade-executor'].upstream_status.unrelated_incident_count, 1);
|
assert.equal(services['trade-executor'].upstream_status.unrelated_incident_count, 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('dashboard service summary exposes executor salt cache and queue delay truth', () => {
|
||||||
|
const config = buildConfig();
|
||||||
|
const dashboard = buildDashboardBootstrap({
|
||||||
|
config,
|
||||||
|
auth: { authenticated: true },
|
||||||
|
portfolioMetric: null,
|
||||||
|
inventorySnapshot: null,
|
||||||
|
marketPrice: null,
|
||||||
|
recentQuotes: [],
|
||||||
|
submissionPage: { page: 1, page_size: 20, total: 0, total_pages: 1, items: [] },
|
||||||
|
submissionSummary: { total: 0, last_submission_at: null },
|
||||||
|
fundingObservations: [],
|
||||||
|
recentDepositStatuses: [],
|
||||||
|
recentTradeDecisions: [],
|
||||||
|
recentExecuteTradeCommands: [],
|
||||||
|
recentExecutionResults: [],
|
||||||
|
recentQuoteOutcomes: [],
|
||||||
|
recentIntentRequests: [],
|
||||||
|
recentAlertTransitions: [],
|
||||||
|
serviceSnapshots: [
|
||||||
|
{
|
||||||
|
service: 'trade-executor',
|
||||||
|
label: 'Trade Executor',
|
||||||
|
base_url: 'http://trade-executor',
|
||||||
|
reachable: true,
|
||||||
|
health: { ok: true },
|
||||||
|
state: {
|
||||||
|
relay: { connected: true, last_message_at: '2026-06-12T10:00:00.000Z' },
|
||||||
|
verifier_salt_cache: {
|
||||||
|
configured: true,
|
||||||
|
has_cached_salt: true,
|
||||||
|
salt_fresh: true,
|
||||||
|
salt_age_ms: 42,
|
||||||
|
last_refresh_completed_at: '2026-06-12T09:59:59.950Z',
|
||||||
|
last_refresh_duration_ms: 25,
|
||||||
|
refresh_in_flight: false,
|
||||||
|
},
|
||||||
|
executor_queue_delay: {
|
||||||
|
status: 'warning',
|
||||||
|
latest_ms: 25,
|
||||||
|
max_ms: 1_250,
|
||||||
|
p90_ms: 1_250,
|
||||||
|
p99_ms: 1_250,
|
||||||
|
},
|
||||||
|
salt_unavailable_reject_count: 1,
|
||||||
|
last_salt_unavailable_at: '2026-06-12T09:59:00.000Z',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
const executor = dashboard.system.service_health.find((service) => service.service === 'trade-executor');
|
||||||
|
assert.equal(executor.summary.verifier_salt_cache.salt_fresh, true);
|
||||||
|
assert.equal(executor.summary.verifier_salt_cache.salt_age_ms, 42);
|
||||||
|
assert.equal(executor.summary.executor_queue_delay.status, 'warning');
|
||||||
|
assert.equal(executor.summary.executor_queue_delay.max_ms, 1_250);
|
||||||
|
assert.equal(executor.summary.salt_unavailable_reject_count, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
test('bootstrap exposes deduped environment status history as environmental conditions', () => {
|
test('bootstrap exposes deduped environment status history as environmental conditions', () => {
|
||||||
const config = buildConfig();
|
const config = buildConfig();
|
||||||
|
|
|
||||||
|
|
@ -42,3 +42,9 @@ test('ops sentinel turns quote lifecycle retention health into storage alerts',
|
||||||
assert.match(source, /writerState\.retention_mode/);
|
assert.match(source, /writerState\.retention_mode/);
|
||||||
assert.match(source, /writerState\.retention_summary/);
|
assert.match(source, /writerState\.retention_summary/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('ops sentinel turns executor salt and queue state into runtime alerts', () => {
|
||||||
|
assert.match(source, /buildExecutorRuntimeAlerts/);
|
||||||
|
assert.match(source, /alerts\.push\(\.\.\.buildExecutorRuntimeAlerts/);
|
||||||
|
assert.match(source, /executor,\s*thresholds,\s*now,\s*pair: config\.activePair/);
|
||||||
|
});
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import test from 'node:test';
|
||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
buildExecutorRuntimeAlerts,
|
||||||
buildMakerCompetitivenessRuntimeAlerts,
|
buildMakerCompetitivenessRuntimeAlerts,
|
||||||
deriveServiceHealth,
|
deriveServiceHealth,
|
||||||
shouldContainExecutorForAlerts,
|
shouldContainExecutorForAlerts,
|
||||||
|
|
@ -158,3 +159,83 @@ test('history writer retention pressure and stale rollups surface as runtime war
|
||||||
assert.equal(stale.label, 'rollup stale');
|
assert.equal(stale.label, 'rollup stale');
|
||||||
assert.ok(stale.reasons.some((reason) => /fail-closed/.test(reason)));
|
assert.ok(stale.reasons.some((reason) => /fail-closed/.test(reason)));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('trade executor salt and queue state surface as runtime health warnings', () => {
|
||||||
|
const health = deriveServiceHealth({
|
||||||
|
service: 'trade-executor',
|
||||||
|
snapshot: {
|
||||||
|
reachable: true,
|
||||||
|
state: {
|
||||||
|
armed: true,
|
||||||
|
verifier_salt_cache: {
|
||||||
|
configured: true,
|
||||||
|
has_cached_salt: false,
|
||||||
|
salt_fresh: false,
|
||||||
|
},
|
||||||
|
executor_queue_delay: {
|
||||||
|
status: 'warning',
|
||||||
|
warning_reason: 'recent command_to_executor_ms max 1500ms exceeds 1000ms',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
health: { ok: true },
|
||||||
|
},
|
||||||
|
now: '2026-06-12T10:00:00.000Z',
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(health.status, 'warning');
|
||||||
|
assert.ok(health.reasons.includes('verifier salt cache is stale or unavailable'));
|
||||||
|
assert.ok(health.reasons.includes('recent command_to_executor_ms max 1500ms exceeds 1000ms'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('executor runtime alerts cover stale salt, refresh failure, and queue delay', () => {
|
||||||
|
const alerts = buildExecutorRuntimeAlerts({
|
||||||
|
executor: {
|
||||||
|
reachable: true,
|
||||||
|
state: {
|
||||||
|
verifier_salt_cache: {
|
||||||
|
configured: true,
|
||||||
|
has_cached_salt: true,
|
||||||
|
salt_fresh: false,
|
||||||
|
salt_age_ms: 750,
|
||||||
|
max_age_ms: 500,
|
||||||
|
refresh_in_flight: false,
|
||||||
|
last_refresh_started_at: '2026-06-12T09:59:59.000Z',
|
||||||
|
last_refresh_completed_at: '2026-06-12T09:59:59.100Z',
|
||||||
|
last_refresh_duration_ms: 100,
|
||||||
|
last_refresh_error: {
|
||||||
|
name: 'Error',
|
||||||
|
message: 'rpc unavailable',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
executor_queue_delay: {
|
||||||
|
status: 'critical',
|
||||||
|
latest_ms: 200,
|
||||||
|
max_ms: 7_500,
|
||||||
|
p90_ms: 5_200,
|
||||||
|
p99_ms: 7_500,
|
||||||
|
sample_count: 10,
|
||||||
|
latest_command_id: 'cmd-1',
|
||||||
|
latest_quote_id: 'quote-1',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
thresholds: {
|
||||||
|
executorQueueDelayWarningMs: 1_000,
|
||||||
|
executorQueueDelayCriticalMs: 5_000,
|
||||||
|
},
|
||||||
|
pair: 'nbtc->eure',
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
alerts.map((alert) => alert.alert_code),
|
||||||
|
[
|
||||||
|
'trade_executor_verifier_salt_unavailable',
|
||||||
|
'trade_executor_verifier_salt_refresh_failed',
|
||||||
|
'trade_executor_command_queue_delay_high',
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert.equal(alerts[0].severity, 'warning');
|
||||||
|
assert.equal(alerts[1].details.last_refresh_error.message, 'rpc unavailable');
|
||||||
|
assert.equal(alerts[2].severity, 'critical');
|
||||||
|
assert.equal(alerts[2].details.p90_ms, 5_200);
|
||||||
|
});
|
||||||
|
|
|
||||||
|
|
@ -19,8 +19,20 @@ test('sentinel snapshot summary keeps trade-executor health fields and drops ret
|
||||||
paused: false,
|
paused: false,
|
||||||
armed: true,
|
armed: true,
|
||||||
relay: { connected: true, last_message_at: '2026-05-06T15:00:00.000Z' },
|
relay: { connected: true, last_message_at: '2026-05-06T15:00:00.000Z' },
|
||||||
|
verifier_salt_cache: {
|
||||||
|
salt_fresh: true,
|
||||||
|
salt_age_ms: 42,
|
||||||
|
refresh_in_flight: false,
|
||||||
|
},
|
||||||
|
executor_queue_delay: {
|
||||||
|
status: 'warning',
|
||||||
|
max_ms: 1500,
|
||||||
|
p90_ms: 1200,
|
||||||
|
},
|
||||||
last_result: { status: 'submitted' },
|
last_result: { status: 'submitted' },
|
||||||
submitted_count: 100,
|
submitted_count: 100,
|
||||||
|
salt_unavailable_reject_count: 2,
|
||||||
|
last_salt_unavailable_at: '2026-05-06T15:00:01.000Z',
|
||||||
processed_idempotency_keys: Object.fromEntries(Array.from({ length: 100 }, (_, index) => [`quote:${index}`, true])),
|
processed_idempotency_keys: Object.fromEntries(Array.from({ length: 100 }, (_, index) => [`quote:${index}`, true])),
|
||||||
submitted_commands: {
|
submitted_commands: {
|
||||||
'cmd-1': { quote_id: 'quote-1', raw_response: { large: true } },
|
'cmd-1': { quote_id: 'quote-1', raw_response: { large: true } },
|
||||||
|
|
@ -30,6 +42,9 @@ test('sentinel snapshot summary keeps trade-executor health fields and drops ret
|
||||||
|
|
||||||
assert.equal(summary.state.armed, true);
|
assert.equal(summary.state.armed, true);
|
||||||
assert.deepEqual(summary.state.relay, { connected: true, last_message_at: '2026-05-06T15:00:00.000Z' });
|
assert.deepEqual(summary.state.relay, { connected: true, last_message_at: '2026-05-06T15:00:00.000Z' });
|
||||||
|
assert.equal(summary.state.verifier_salt_cache.salt_fresh, true);
|
||||||
|
assert.equal(summary.state.executor_queue_delay.status, 'warning');
|
||||||
|
assert.equal(summary.state.salt_unavailable_reject_count, 2);
|
||||||
assert.equal(summary.state.processed_idempotency_keys, undefined);
|
assert.equal(summary.state.processed_idempotency_keys, undefined);
|
||||||
assert.equal(summary.state.submitted_commands, undefined);
|
assert.equal(summary.state.submitted_commands, undefined);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,7 @@ test('trade executor records hot path timing in result payloads', () => {
|
||||||
assert.match(source, /node:perf_hooks/);
|
assert.match(source, /node:perf_hooks/);
|
||||||
assert.match(source, /startExecutorTiming\(event\)/);
|
assert.match(source, /startExecutorTiming\(event\)/);
|
||||||
assert.match(source, /executor_timing/);
|
assert.match(source, /executor_timing/);
|
||||||
|
assert.match(source, /command_to_executor_ms/);
|
||||||
assert.match(source, /current_salt_ms/);
|
assert.match(source, /current_salt_ms/);
|
||||||
assert.match(source, /sign_ms/);
|
assert.match(source, /sign_ms/);
|
||||||
assert.match(source, /relay_response_ms/);
|
assert.match(source, /relay_response_ms/);
|
||||||
|
|
@ -40,10 +41,31 @@ test('trade executor records hot path timing in result payloads', () => {
|
||||||
test('trade executor uses bounded verifier salt cache instead of per-command salt RPC', () => {
|
test('trade executor uses bounded verifier salt cache instead of per-command salt RPC', () => {
|
||||||
assert.match(source, /createVerifierSaltCache/);
|
assert.match(source, /createVerifierSaltCache/);
|
||||||
assert.match(source, /verifierSaltCache\.start\(\)/);
|
assert.match(source, /verifierSaltCache\.start\(\)/);
|
||||||
assert.match(source, /verifierSaltCache\.getFreshSalt\(\)/);
|
assert.match(source, /verifierSaltCache\.getCachedFreshSalt\(\)/);
|
||||||
|
assert.match(source, /verifier_salt_unavailable/);
|
||||||
|
assert.match(source, /fresh verifier salt unavailable; command was not signed or relayed/);
|
||||||
assert.match(source, /current_salt_source/);
|
assert.match(source, /current_salt_source/);
|
||||||
assert.match(source, /current_salt_age_ms/);
|
assert.match(source, /current_salt_age_ms/);
|
||||||
|
assert.match(source, /current_salt_unavailable_reason/);
|
||||||
assert.match(source, /verifier_salt_cache: verifierSaltCache\.getState\(\)/);
|
assert.match(source, /verifier_salt_cache: verifierSaltCache\.getState\(\)/);
|
||||||
assert.match(source, /verifierSaltCache\.stop\(\)/);
|
assert.match(source, /verifierSaltCache\.stop\(\)/);
|
||||||
|
assert.doesNotMatch(source, /verifierSaltCache\.getFreshSalt\(\)/);
|
||||||
assert.doesNotMatch(source, /await verifierClient\.currentSalt\(\);/);
|
assert.doesNotMatch(source, /await verifierClient\.currentSalt\(\);/);
|
||||||
|
|
||||||
|
const expiryIndex = source.indexOf('const expiry = classifyExecuteCommandExpiry(event);');
|
||||||
|
const saltLookupIndex = source.indexOf('verifierSaltCache.getCachedFreshSalt()');
|
||||||
|
const saltUnavailableIndex = source.indexOf("result_code: 'verifier_salt_unavailable'");
|
||||||
|
const signIndex = source.indexOf('buildQuoteResponseSubmission({');
|
||||||
|
const relayIndex = source.indexOf("relayClient.request('quote_response'");
|
||||||
|
assert.ok(expiryIndex > 0 && expiryIndex < saltLookupIndex);
|
||||||
|
assert.ok(saltUnavailableIndex > 0 && saltUnavailableIndex < signIndex);
|
||||||
|
assert.ok(saltUnavailableIndex < relayIndex);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('trade executor exposes queue delay and salt rejection state', () => {
|
||||||
|
assert.match(source, /createExecutorQueueDelayTracker/);
|
||||||
|
assert.match(source, /recordExecutorQueueDelay\(timing, payload\)/);
|
||||||
|
assert.match(source, /executor_queue_delay: executorQueueDelayTracker\.getState\(\)/);
|
||||||
|
assert.match(source, /salt_unavailable_reject_count/);
|
||||||
|
assert.match(source, /last_salt_unavailable_at/);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
|
||||||
|
|
||||||
import { createVerifierSaltCache } from '../src/core/verifier-salt-cache.mjs';
|
import { createVerifierSaltCache } from '../src/core/verifier-salt-cache.mjs';
|
||||||
|
|
||||||
test('verifier salt cache reuses a fresh salt without calling the verifier hot path', async () => {
|
test('verifier salt cache cached fresh read returns without calling the verifier loader', async () => {
|
||||||
let nowMs = 1_000;
|
let nowMs = 1_000;
|
||||||
let calls = 0;
|
let calls = 0;
|
||||||
const cache = createVerifierSaltCache({
|
const cache = createVerifierSaltCache({
|
||||||
|
|
@ -15,37 +15,101 @@ test('verifier salt cache reuses a fresh salt without calling the verifier hot p
|
||||||
now: () => nowMs,
|
now: () => nowMs,
|
||||||
});
|
});
|
||||||
|
|
||||||
const refreshed = await cache.getFreshSalt();
|
const refreshed = await cache.refreshNow();
|
||||||
nowMs += 100;
|
nowMs += 100;
|
||||||
const cached = await cache.getFreshSalt();
|
const cached = cache.getCachedFreshSalt();
|
||||||
|
|
||||||
assert.equal(calls, 1);
|
assert.equal(calls, 1);
|
||||||
assert.equal(refreshed.source, 'refresh');
|
assert.equal(refreshed.source, 'refresh');
|
||||||
assert.equal(cached.source, 'cache');
|
assert.equal(cached.source, 'cache');
|
||||||
|
assert.equal(cached.available, true);
|
||||||
assert.equal(cached.currentSaltHex, '252812b3');
|
assert.equal(cached.currentSaltHex, '252812b3');
|
||||||
assert.equal(cached.ageMs, 100);
|
assert.equal(cached.ageMs, 100);
|
||||||
|
assert.equal(cache.getState().salt_fresh, true);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('verifier salt cache refreshes a stale salt before returning it', async () => {
|
test('verifier salt cache missing or stale cached read returns unavailable without loader call', async () => {
|
||||||
let nowMs = 1_000;
|
let nowMs = 1_000;
|
||||||
const salts = ['252812b3', '252812b4'];
|
let calls = 0;
|
||||||
const cache = createVerifierSaltCache({
|
const cache = createVerifierSaltCache({
|
||||||
loadSalt: async () => salts.shift(),
|
loadSalt: async () => {
|
||||||
|
calls += 1;
|
||||||
|
return '252812b3';
|
||||||
|
},
|
||||||
maxAgeMs: 500,
|
maxAgeMs: 500,
|
||||||
now: () => nowMs,
|
now: () => nowMs,
|
||||||
});
|
});
|
||||||
|
|
||||||
const first = await cache.getFreshSalt();
|
const missing = cache.getCachedFreshSalt();
|
||||||
nowMs += 501;
|
assert.equal(missing.available, false);
|
||||||
const second = await cache.getFreshSalt();
|
assert.equal(missing.reason, 'empty_cache');
|
||||||
|
assert.equal(calls, 0);
|
||||||
|
|
||||||
assert.equal(first.currentSaltHex, '252812b3');
|
await cache.refreshNow();
|
||||||
assert.equal(second.currentSaltHex, '252812b4');
|
nowMs += 501;
|
||||||
assert.equal(second.source, 'refresh');
|
const stale = cache.getCachedFreshSalt();
|
||||||
assert.equal(second.ageMs, 0);
|
|
||||||
|
assert.equal(stale.available, false);
|
||||||
|
assert.equal(stale.reason, 'stale_cache');
|
||||||
|
assert.equal(stale.ageMs, 501);
|
||||||
|
assert.equal(calls, 1);
|
||||||
|
assert.equal(cache.getState().salt_fresh, false);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('verifier salt cache fails closed when stale salt refresh fails', async () => {
|
test('verifier salt cache cached read does not wait for slow background current_salt refresh', async () => {
|
||||||
|
let calls = 0;
|
||||||
|
let resolveSalt;
|
||||||
|
const slowSalt = new Promise((resolve) => {
|
||||||
|
resolveSalt = resolve;
|
||||||
|
});
|
||||||
|
const cache = createVerifierSaltCache({
|
||||||
|
loadSalt: async () => {
|
||||||
|
calls += 1;
|
||||||
|
return slowSalt;
|
||||||
|
},
|
||||||
|
setIntervalFn: () => ({ unref() {} }),
|
||||||
|
clearIntervalFn: () => {},
|
||||||
|
});
|
||||||
|
|
||||||
|
cache.start();
|
||||||
|
await Promise.resolve();
|
||||||
|
|
||||||
|
const cached = cache.getCachedFreshSalt();
|
||||||
|
assert.equal(cached.available, false);
|
||||||
|
assert.equal(cached.reason, 'empty_cache');
|
||||||
|
assert.equal(cached.state.refresh_in_flight, true);
|
||||||
|
assert.equal(calls, 1);
|
||||||
|
|
||||||
|
resolveSalt('252812b3');
|
||||||
|
await cache.refreshNow();
|
||||||
|
assert.equal(cache.getCachedFreshSalt().available, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('verifier salt cache background refresh records success duration and state', async () => {
|
||||||
|
let nowMs = 1_000;
|
||||||
|
const cache = createVerifierSaltCache({
|
||||||
|
loadSalt: async () => {
|
||||||
|
nowMs += 37;
|
||||||
|
return '252812b3';
|
||||||
|
},
|
||||||
|
maxAgeMs: 500,
|
||||||
|
now: () => nowMs,
|
||||||
|
});
|
||||||
|
|
||||||
|
const refreshed = await cache.refreshNow({ reason: 'unit_test' });
|
||||||
|
const state = cache.getState();
|
||||||
|
|
||||||
|
assert.equal(refreshed.currentSaltHex, '252812b3');
|
||||||
|
assert.equal(state.salt_fresh, true);
|
||||||
|
assert.equal(state.last_refresh_started_at, '1970-01-01T00:00:01.000Z');
|
||||||
|
assert.equal(state.last_refresh_completed_at, '1970-01-01T00:00:01.037Z');
|
||||||
|
assert.equal(state.last_refresh_duration_ms, 37);
|
||||||
|
assert.equal(state.last_refresh_reason, 'unit_test');
|
||||||
|
assert.equal(state.refresh_in_flight, false);
|
||||||
|
assert.equal(state.last_refresh_error, null);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('verifier salt cache refresh failure preserves still-fresh cached salt', async () => {
|
||||||
let nowMs = 1_000;
|
let nowMs = 1_000;
|
||||||
let fail = false;
|
let fail = false;
|
||||||
const cache = createVerifierSaltCache({
|
const cache = createVerifierSaltCache({
|
||||||
|
|
@ -57,15 +121,19 @@ test('verifier salt cache fails closed when stale salt refresh fails', async ()
|
||||||
now: () => nowMs,
|
now: () => nowMs,
|
||||||
});
|
});
|
||||||
|
|
||||||
await cache.getFreshSalt();
|
await cache.refreshNow();
|
||||||
nowMs += 501;
|
nowMs += 100;
|
||||||
fail = true;
|
fail = true;
|
||||||
|
|
||||||
await assert.rejects(
|
await assert.rejects(
|
||||||
() => cache.getFreshSalt(),
|
() => cache.refreshNow({ reason: 'prefetch' }),
|
||||||
/rpc unavailable/,
|
/rpc unavailable/,
|
||||||
);
|
);
|
||||||
assert.equal(cache.getState().fresh, false);
|
const cached = cache.getCachedFreshSalt();
|
||||||
|
assert.equal(cached.available, true);
|
||||||
|
assert.equal(cached.currentSaltHex, '252812b3');
|
||||||
|
assert.equal(cached.ageMs, 100);
|
||||||
|
assert.equal(cache.getState().fresh, true);
|
||||||
assert.equal(cache.getState().last_refresh_error.message, 'rpc unavailable');
|
assert.equal(cache.getState().last_refresh_error.message, 'rpc unavailable');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -75,8 +143,9 @@ test('verifier salt cache rejects malformed salts before signing can use them',
|
||||||
});
|
});
|
||||||
|
|
||||||
await assert.rejects(
|
await assert.rejects(
|
||||||
() => cache.getFreshSalt(),
|
() => cache.refreshNow(),
|
||||||
/current_salt must be 4 bytes in hex/,
|
/current_salt must be 4 bytes in hex/,
|
||||||
);
|
);
|
||||||
assert.equal(cache.getState().has_cached_salt, false);
|
assert.equal(cache.getState().has_cached_salt, false);
|
||||||
|
assert.equal(cache.getCachedFreshSalt().available, false);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue