export const SERVICE_HEALTH_LEVELS = ['healthy', 'warning', 'critical', 'offline', 'paused']; export const DEFAULT_EXECUTOR_QUEUE_DELAY_WARNING_MS = 1_000; export const DEFAULT_EXECUTOR_QUEUE_DELAY_CRITICAL_MS = 5_000; const HEALTH_RANK = { healthy: 0, warning: 1, critical: 2, offline: 3, paused: 1, }; const ALERT_RANK = { info: 0, warning: 1, critical: 2, }; export function createRuntimeHealthThresholds(config = {}) { return { ingestMessageStaleMs: Number(config.opsSentinelIngestMessageStaleMs || 30_000), ingestQuoteStaleMs: Number(config.opsSentinelIngestQuoteStaleMs || 30_000), ingestPublishStaleMs: Number(config.opsSentinelIngestPublishStaleMs || 30_000), executorRelayStaleMs: Number(config.opsSentinelExecutorRelayStaleMs || 30_000), historyWriterStaleMs: Number(config.opsSentinelHistoryWriterStaleMs || 45_000), dashboardSourceDegradedMs: Number(config.opsSentinelDashboardSourceDegradedMs || 30_000), sentinelStaleMs: Number(config.opsSentinelSelfStaleMs || 20_000), anomalyWindowSize: Number(config.opsSentinelAnomalyWindowSize || 6), anomalyQuoteRateCollapseRatio: Number(config.opsSentinelAnomalyQuoteRateCollapseRatio || 0.25), anomalyReconnectSpikeMultiplier: Number(config.opsSentinelAnomalyReconnectSpikeMultiplier || 2), containmentCooldownMs: Number(config.opsSentinelContainmentCooldownMs || 60_000), executorQueueDelayWarningMs: Number( config.executorQueueDelayWarningMs || DEFAULT_EXECUTOR_QUEUE_DELAY_WARNING_MS, ), executorQueueDelayCriticalMs: Number( config.executorQueueDelayCriticalMs || DEFAULT_EXECUTOR_QUEUE_DELAY_CRITICAL_MS, ), }; } export function evaluateRuntimeHealth({ servicesByName, activePair, activePairs = activePair ? [activePair] : [], activeAlerts = [], now = new Date().toISOString(), } = {}) { const serviceHealth = new Map(); const alertIndex = indexAlertsByService(activeAlerts); for (const [service, snapshot] of Object.entries(servicesByName || {})) { const alerts = alertIndex.get(service) || []; serviceHealth.set(service, deriveServiceHealth({ service, snapshot, activePair, activePairs, activeAlerts: alerts, now, })); } return serviceHealth; } export function deriveServiceHealth({ service, snapshot, activePair = null, activePairs = activePair ? [activePair] : [], activeAlerts = [], now = new Date().toISOString(), } = {}) { const state = snapshot?.state || {}; const health = snapshot?.health || {}; const reachable = snapshot?.reachable !== false; const paused = state.paused ?? health.paused ?? false; const highestAlertSeverity = highestAlertSeverityForService(activeAlerts); const freshnessAt = inferServiceFreshnessTimestamp(service, state, health); const freshnessAgeMs = ageMs(freshnessAt, now); const reasons = []; let status = paused ? 'paused' : reachable ? 'healthy' : 'offline'; let label = status; if (!reachable) { reasons.push('service unreachable'); label = 'offline'; } if (service === 'near-intents-ingest') { const ingestClassification = classifyNearIntentsIngestHealth({ state, health, activeAlerts, reachable, now, }); status = escalateHealth(status, ingestClassification.status); label = ingestClassification.label; reasons.push(...ingestClassification.reasons); } else { if (health.ok === false && reachable) { status = escalateHealth(status, 'critical'); reasons.push(health.reason || 'service health check failed'); } if (highestAlertSeverity === 'critical') { status = escalateHealth(status, 'critical'); reasons.push(`critical alert active (${activeAlerts[0]?.alert_code || 'runtime'})`); } else if (highestAlertSeverity === 'warning') { status = escalateHealth(status, 'warning'); reasons.push(`warning alert active (${activeAlerts[0]?.alert_code || 'runtime'})`); } } if (service !== 'near-intents-ingest' && status === 'healthy') { label = 'healthy'; } if (service !== 'near-intents-ingest' && status === 'paused') { label = 'paused'; } if (service !== 'near-intents-ingest' && status === 'critical' && label === 'critical') { label = 'critical'; } if (service !== 'near-intents-ingest' && status === 'warning' && label === 'warning') { label = 'warning'; } if (service !== 'near-intents-ingest' && status === 'offline') { label = 'offline'; } if (service === 'trade-executor' && state.relay?.connected === false) { status = escalateHealth(status, 'critical'); label = 'relay disconnected'; if (!reasons.includes('solver relay disconnected')) { reasons.push('solver relay disconnected'); } } if (service === 'trade-executor') { const saltState = state.verifier_salt_cache || health.verifier_salt_cache || null; const queueDelay = state.executor_queue_delay || health.executor_queue_delay || null; if (saltState?.configured && saltState.salt_fresh !== true) { status = escalateHealth(status, 'warning'); if (label === 'healthy') label = 'salt unavailable'; reasons.push('verifier salt cache is stale or unavailable'); } if (saltState?.last_refresh_error) { status = escalateHealth(status, 'warning'); if (label === 'healthy') label = 'salt refresh failed'; reasons.push('verifier salt refresh failed'); } if (queueDelay?.status === 'critical') { status = escalateHealth(status, 'critical'); label = 'queue delayed'; reasons.push(queueDelay.warning_reason || 'executor queue delay is critical'); } else if (queueDelay?.status === 'warning') { status = escalateHealth(status, 'warning'); if (label === 'healthy') label = 'queue delayed'; reasons.push(queueDelay.warning_reason || 'executor queue delay is high'); } } if (service === 'history-writer') { if (state.database_connectivity === false) { status = escalateHealth(status, 'critical'); label = 'database disconnected'; reasons.push('database connectivity failed'); } else if (freshnessAgeMs != null && freshnessAgeMs > 45_000) { status = escalateHealth(status, 'warning'); label = 'writer lagging'; reasons.push('writer freshness degraded'); } if (state.retention_mode === 'blocked_invalid_policy') { status = escalateHealth(status, 'warning'); label = 'retention blocked'; reasons.push('quote lifecycle retention policy is invalid or missing'); } else if (state.retention_mode === 'blocked_rollup_stale') { status = escalateHealth(status, 'warning'); label = 'rollup stale'; reasons.push('quote lifecycle rollup is stale, pruning is fail-closed'); } else if (state.retention_mode === 'pressure') { status = escalateHealth(status, 'warning'); label = 'storage pressure'; reasons.push('quote lifecycle storage pressure retention mode is active'); } } if (service === 'operator-dashboard') { if ((state.source_error_count || 0) > 0 || (health.source_error_count || 0) > 0) { status = escalateHealth(status, 'warning'); label = 'sources degraded'; reasons.push('dashboard source degraded'); } } if ( ['strategy-engine', 'trade-executor'].includes(service) && (state.armed ?? false) && hasCriticalTruthAlert(activeAlerts, activePairs.length ? activePairs : activePair) ) { status = escalateHealth(status, 'critical'); if (label === 'healthy' || label === 'warning') { label = 'armed on stale truth'; } reasons.push('armed while critical upstream truth is stale'); } return { service, status, label, reachable, paused, armed: state.armed ?? null, health_ok: status === 'healthy' || status === 'paused', highest_alert_severity: highestAlertSeverity, reasons: dedupeReasons(reasons), freshness_at: freshnessAt, freshness_age_ms: freshnessAgeMs, }; } function classifyNearIntentsIngestHealth({ state, health, activeAlerts, reachable, now, }) { const reasons = []; const alertCodes = new Set((activeAlerts || []).map((alert) => alert.alert_code)); const connected = state.ingest?.connected ?? health.connected ?? null; const matchingAgeMs = ageMs(state.ingest?.last_matching_quote_at, now); const publishedAgeMs = ageMs(state.ingest?.last_published_at, now); if (!reachable) { return { status: 'offline', label: 'offline', reasons: ['service unreachable'], }; } if (connected === false || alertCodes.has('near_intents_ingest_disconnected')) { return { status: 'critical', label: 'disconnected', reasons: ['websocket disconnected', 'critical alert active (near_intents_ingest_disconnected)'], }; } if ( alertCodes.has('near_intents_publish_stale') || ( state.ingest?.last_matching_quote_at && state.ingest?.last_published_at && matchingAgeMs != null && publishedAgeMs != null && publishedAgeMs > matchingAgeMs + 5_000 ) ) { return { status: 'critical', label: 'publish stalled', reasons: ['quote publish path stalled', 'critical alert active (near_intents_publish_stale)'], }; } if (alertCodes.has('near_intents_quotes_stale') || health.reason === 'quote truth stale') { reasons.push('connected, no recent quotes for active pair'); if (matchingAgeMs != null) { reasons.push(`last matching quote ${matchingAgeMs}ms ago`); } return { status: 'warning', label: 'no recent quotes', reasons, }; } return { status: health.ok === false ? 'warning' : 'healthy', label: connected === true ? 'healthy' : 'unknown', reasons: health.ok === false ? [health.reason || 'service health degraded'] : [], }; } function dedupeReasons(reasons) { return [...new Set((reasons || []).filter(Boolean))]; } export function inferServiceFreshnessTimestamp(service, state = {}, health = {}) { switch (service) { case 'near-intents-ingest': return ( state.ingest?.last_published_at || state.ingest?.last_matching_quote_at || state.ingest?.last_message_at || null ); case 'market-reference-ingest': return state.last_published_at || null; case 'inventory-sync': return state.last_sync_at || null; case 'liquidity-manager': return state.last_refresh_at || null; case 'history-writer': return state.last_write_at || state.last_metrics_at || null; case 'ops-sentinel': return state.last_runtime_eval_at || state.last_evaluated_at || health.last_event_at || null; case 'strategy-engine': return state.latest_decision?.decision_at || state.latest_inventory_event?.ingested_at || null; case 'trade-executor': return state.relay?.last_message_at || state.last_quote_status?.created_at || null; case 'operator-dashboard': return state.last_bootstrap_at || state.last_source_error_at || null; default: return null; } } export function buildRuntimeAlert({ alert_code, severity, reason, service_scope, pair = null, asset_id = null, tx_hash = null, details = {}, }) { return { alert_code, severity, reason, service_scope, pair, asset_id, tx_hash, details, }; } export function buildMakerCompetitivenessRuntimeAlerts({ makerCompetitiveness, minPairSamples = 5, minQuoteNotFoundOrFinishedRate = 0.4, } = {}) { const groups = Array.isArray(makerCompetitiveness?.groups) ? makerCompetitiveness.groups : []; const pairStats = new Map(); for (const group of groups) { const pair = group.pair || null; if (!pair) continue; const current = pairStats.get(pair) || { pair, sample_count: 0, quote_not_found_or_finished_count: 0, accepted_count: 0, policy_skip_count: 0, worst_group: null, }; const count = Number(group.count || 0); const staleFinishedCount = group.failure_category === 'quote_not_found_or_finished' ? count : 0; current.sample_count += count; current.quote_not_found_or_finished_count += staleFinishedCount; current.accepted_count += Number(group.accepted_count || 0); current.policy_skip_count += Number(group.policy_skip_count || 0); if ( staleFinishedCount > 0 && (!current.worst_group || staleFinishedCount > Number(current.worst_group.count || 0)) ) { current.worst_group = group; } pairStats.set(pair, current); } return [...pairStats.values()] .filter((stats) => { if (stats.sample_count < minPairSamples) return false; if (stats.quote_not_found_or_finished_count <= 0) return false; return stats.quote_not_found_or_finished_count / stats.sample_count >= minQuoteNotFoundOrFinishedRate; }) .map((stats) => { const rate = stats.quote_not_found_or_finished_count / stats.sample_count; return buildRuntimeAlert({ alert_code: 'maker_quote_not_found_or_finished_rate_high', severity: 'warning', reason: `maker responses for ${stats.pair} are frequently reaching relay after the quote is finished`, service_scope: 'strategy-engine', pair: stats.pair, details: { generated_at: makerCompetitiveness?.generated_at || null, sample_count: stats.sample_count, quote_not_found_or_finished_count: stats.quote_not_found_or_finished_count, quote_not_found_or_finished_rate: Number(rate.toFixed(4)), accepted_count: stats.accepted_count, policy_skip_count: stats.policy_skip_count, min_pair_samples: minPairSamples, min_quote_not_found_or_finished_rate: minQuoteNotFoundOrFinishedRate, worst_direction: stats.worst_group?.direction || null, worst_request_kind: stats.worst_group?.request_kind || null, worst_quote_age_bucket: stats.worst_group?.quote_age_bucket || null, worst_notional_bucket: stats.worst_group?.notional_bucket || null, }, }); }); } export function buildExecutorRuntimeAlerts({ executor, thresholds = createRuntimeHealthThresholds(), now = new Date().toISOString(), pair = null, } = {}) { void now; const alerts = []; const state = executor?.state || {}; const health = executor?.health || {}; const saltState = state.verifier_salt_cache || health.verifier_salt_cache || null; const queueDelay = state.executor_queue_delay || health.executor_queue_delay || null; if (saltState?.configured && saltState.salt_fresh !== true) { alerts.push(buildRuntimeAlert({ alert_code: 'trade_executor_verifier_salt_unavailable', severity: 'warning', reason: saltState.has_cached_salt ? 'trade-executor verifier salt cache is stale' : 'trade-executor verifier salt cache is unavailable', service_scope: 'trade-executor', pair, details: { has_cached_salt: saltState.has_cached_salt ?? null, salt_fresh: saltState.salt_fresh ?? saltState.fresh ?? null, salt_age_ms: saltState.salt_age_ms ?? saltState.cached_age_ms ?? null, max_age_ms: saltState.max_age_ms ?? null, refresh_in_flight: saltState.refresh_in_flight ?? null, last_refresh_started_at: saltState.last_refresh_started_at || null, last_refresh_completed_at: saltState.last_refresh_completed_at || null, last_refresh_error: saltState.last_refresh_error || null, }, })); } if (saltState?.last_refresh_error) { alerts.push(buildRuntimeAlert({ alert_code: 'trade_executor_verifier_salt_refresh_failed', severity: 'warning', reason: 'trade-executor verifier salt refresh failed', service_scope: 'trade-executor', pair, details: { last_refresh_error: saltState.last_refresh_error, last_refresh_started_at: saltState.last_refresh_started_at || null, last_refresh_completed_at: saltState.last_refresh_completed_at || null, last_refresh_duration_ms: saltState.last_refresh_duration_ms ?? null, refresh_in_flight: saltState.refresh_in_flight ?? null, has_cached_salt: saltState.has_cached_salt ?? null, salt_fresh: saltState.salt_fresh ?? saltState.fresh ?? null, }, })); } const queueMaxMs = Number(queueDelay?.max_ms); if (Number.isFinite(queueMaxMs) && queueMaxMs >= thresholds.executorQueueDelayWarningMs) { const critical = queueMaxMs >= thresholds.executorQueueDelayCriticalMs; alerts.push(buildRuntimeAlert({ alert_code: 'trade_executor_command_queue_delay_high', severity: critical ? 'critical' : 'warning', reason: `trade-executor command queue delay ${queueMaxMs}ms exceeds ${ critical ? thresholds.executorQueueDelayCriticalMs : thresholds.executorQueueDelayWarningMs }ms`, service_scope: 'trade-executor', pair, details: { status: queueDelay.status || (critical ? 'critical' : 'warning'), latest_ms: queueDelay.latest_ms ?? null, max_ms: queueDelay.max_ms ?? null, p50_ms: queueDelay.p50_ms ?? null, p90_ms: queueDelay.p90_ms ?? null, p99_ms: queueDelay.p99_ms ?? null, sample_count: queueDelay.sample_count ?? null, warning_after_ms: thresholds.executorQueueDelayWarningMs, critical_after_ms: thresholds.executorQueueDelayCriticalMs, latest_command_id: queueDelay.latest_command_id || null, latest_quote_id: queueDelay.latest_quote_id || null, }, })); } return alerts; } export function shouldRaiseIngestPublishStale({ lastMatchingQuoteAt = null, lastPublishedAt = null, matchingQuoteAgeMs = null, publishedAgeMs = null, publishStaleMs, } = {}) { if (!lastMatchingQuoteAt) return false; if (!lastPublishedAt) return true; if (publishedAgeMs == null || publishedAgeMs > publishStaleMs) return true; if ( matchingQuoteAgeMs != null && matchingQuoteAgeMs <= publishStaleMs && publishedAgeMs > matchingQuoteAgeMs + 5_000 ) { return true; } return false; } export function shouldContainExecutorForAlerts(alerts = []) { void alerts; return false; } export function ageMs(value, now = new Date().toISOString()) { if (!value) return null; const left = new Date(value).getTime(); const right = new Date(now).getTime(); if (!Number.isFinite(left) || !Number.isFinite(right)) return null; return Math.max(0, right - left); } function highestAlertSeverityForService(alerts) { let highest = null; let highestRank = -1; for (const alert of alerts || []) { const rank = ALERT_RANK[alert.severity] ?? -1; if (rank > highestRank) { highest = alert.severity; highestRank = rank; } } return highest; } function indexAlertsByService(activeAlerts) { const index = new Map(); for (const alert of activeAlerts || []) { const list = index.get(alert.service_scope) || []; list.push(alert); index.set(alert.service_scope, list); } return index; } function hasCriticalTruthAlert(alerts, activePair) { const activePairSet = new Set(Array.isArray(activePair) ? activePair : [activePair].filter(Boolean)); return (alerts || []).some((alert) => ( alert.severity === 'critical' && ( alert.pair == null || activePairSet.has(alert.pair) || alert.alert_code.includes('stale') || alert.alert_code.includes('disconnected') ) )); } function escalateHealth(current, next) { const currentRank = HEALTH_RANK[current] ?? 0; const nextRank = HEALTH_RANK[next] ?? 0; return nextRank > currentRank ? next : current; }