ddd3dfb9e2
All checks were successful
deploy / deploy (push) Successful in 58s
Proof: quote-response execution now uses verifierSaltCache.getCachedFreshSalt() only, rejects verifier_salt_unavailable before signing or relay when no fresh cached salt exists, preserves stale command expiry before salt lookup, exposes salt cache and executor queue-delay state in health/sentinel/dashboard surfaces, and is covered by targeted regression tests plus full npm test and dashboard build. Assumptions: the existing verifier salt freshness policy remains safe for cached signing use; fixed in-code queue-delay warning thresholds are sufficient for operator alerts without new latency env vars; no strategy selection, pricing, edge, notional, inventory, arming, signer, or pair enablement behavior changed. Still fake: venue-native terminal fill ids and fee-complete realized PnL remain unavailable; executor concurrency remains single-consumer; live post-deploy evidence still needs collection after repo-workflow deployment.
576 lines
19 KiB
JavaScript
576 lines
19 KiB
JavaScript
export const SERVICE_HEALTH_LEVELS = ['healthy', 'warning', 'critical', 'offline', 'paused'];
|
|
export const DEFAULT_EXECUTOR_QUEUE_DELAY_WARNING_MS = 1_000;
|
|
export const DEFAULT_EXECUTOR_QUEUE_DELAY_CRITICAL_MS = 5_000;
|
|
|
|
const HEALTH_RANK = {
|
|
healthy: 0,
|
|
warning: 1,
|
|
critical: 2,
|
|
offline: 3,
|
|
paused: 1,
|
|
};
|
|
|
|
const ALERT_RANK = {
|
|
info: 0,
|
|
warning: 1,
|
|
critical: 2,
|
|
};
|
|
|
|
export function createRuntimeHealthThresholds(config = {}) {
|
|
return {
|
|
ingestMessageStaleMs: Number(config.opsSentinelIngestMessageStaleMs || 30_000),
|
|
ingestQuoteStaleMs: Number(config.opsSentinelIngestQuoteStaleMs || 30_000),
|
|
ingestPublishStaleMs: Number(config.opsSentinelIngestPublishStaleMs || 30_000),
|
|
executorRelayStaleMs: Number(config.opsSentinelExecutorRelayStaleMs || 30_000),
|
|
historyWriterStaleMs: Number(config.opsSentinelHistoryWriterStaleMs || 45_000),
|
|
dashboardSourceDegradedMs: Number(config.opsSentinelDashboardSourceDegradedMs || 30_000),
|
|
sentinelStaleMs: Number(config.opsSentinelSelfStaleMs || 20_000),
|
|
anomalyWindowSize: Number(config.opsSentinelAnomalyWindowSize || 6),
|
|
anomalyQuoteRateCollapseRatio: Number(config.opsSentinelAnomalyQuoteRateCollapseRatio || 0.25),
|
|
anomalyReconnectSpikeMultiplier: Number(config.opsSentinelAnomalyReconnectSpikeMultiplier || 2),
|
|
containmentCooldownMs: Number(config.opsSentinelContainmentCooldownMs || 60_000),
|
|
executorQueueDelayWarningMs: Number(
|
|
config.executorQueueDelayWarningMs || DEFAULT_EXECUTOR_QUEUE_DELAY_WARNING_MS,
|
|
),
|
|
executorQueueDelayCriticalMs: Number(
|
|
config.executorQueueDelayCriticalMs || DEFAULT_EXECUTOR_QUEUE_DELAY_CRITICAL_MS,
|
|
),
|
|
};
|
|
}
|
|
|
|
export function evaluateRuntimeHealth({
|
|
servicesByName,
|
|
activePair,
|
|
activePairs = activePair ? [activePair] : [],
|
|
activeAlerts = [],
|
|
now = new Date().toISOString(),
|
|
} = {}) {
|
|
const serviceHealth = new Map();
|
|
const alertIndex = indexAlertsByService(activeAlerts);
|
|
|
|
for (const [service, snapshot] of Object.entries(servicesByName || {})) {
|
|
const alerts = alertIndex.get(service) || [];
|
|
serviceHealth.set(service, deriveServiceHealth({
|
|
service,
|
|
snapshot,
|
|
activePair,
|
|
activePairs,
|
|
activeAlerts: alerts,
|
|
now,
|
|
}));
|
|
}
|
|
|
|
return serviceHealth;
|
|
}
|
|
|
|
export function deriveServiceHealth({
|
|
service,
|
|
snapshot,
|
|
activePair = null,
|
|
activePairs = activePair ? [activePair] : [],
|
|
activeAlerts = [],
|
|
now = new Date().toISOString(),
|
|
} = {}) {
|
|
const state = snapshot?.state || {};
|
|
const health = snapshot?.health || {};
|
|
const reachable = snapshot?.reachable !== false;
|
|
const paused = state.paused ?? health.paused ?? false;
|
|
const highestAlertSeverity = highestAlertSeverityForService(activeAlerts);
|
|
const freshnessAt = inferServiceFreshnessTimestamp(service, state, health);
|
|
const freshnessAgeMs = ageMs(freshnessAt, now);
|
|
const reasons = [];
|
|
let status = paused ? 'paused' : reachable ? 'healthy' : 'offline';
|
|
let label = status;
|
|
|
|
if (!reachable) {
|
|
reasons.push('service unreachable');
|
|
label = 'offline';
|
|
}
|
|
|
|
if (service === 'near-intents-ingest') {
|
|
const ingestClassification = classifyNearIntentsIngestHealth({
|
|
state,
|
|
health,
|
|
activeAlerts,
|
|
reachable,
|
|
now,
|
|
});
|
|
status = escalateHealth(status, ingestClassification.status);
|
|
label = ingestClassification.label;
|
|
reasons.push(...ingestClassification.reasons);
|
|
} else {
|
|
if (health.ok === false && reachable) {
|
|
status = escalateHealth(status, 'critical');
|
|
reasons.push(health.reason || 'service health check failed');
|
|
}
|
|
|
|
if (highestAlertSeverity === 'critical') {
|
|
status = escalateHealth(status, 'critical');
|
|
reasons.push(`critical alert active (${activeAlerts[0]?.alert_code || 'runtime'})`);
|
|
} else if (highestAlertSeverity === 'warning') {
|
|
status = escalateHealth(status, 'warning');
|
|
reasons.push(`warning alert active (${activeAlerts[0]?.alert_code || 'runtime'})`);
|
|
}
|
|
}
|
|
|
|
if (service !== 'near-intents-ingest' && status === 'healthy') {
|
|
label = 'healthy';
|
|
}
|
|
if (service !== 'near-intents-ingest' && status === 'paused') {
|
|
label = 'paused';
|
|
}
|
|
if (service !== 'near-intents-ingest' && status === 'critical' && label === 'critical') {
|
|
label = 'critical';
|
|
}
|
|
if (service !== 'near-intents-ingest' && status === 'warning' && label === 'warning') {
|
|
label = 'warning';
|
|
}
|
|
if (service !== 'near-intents-ingest' && status === 'offline') {
|
|
label = 'offline';
|
|
}
|
|
|
|
if (service === 'trade-executor' && state.relay?.connected === false) {
|
|
status = escalateHealth(status, 'critical');
|
|
label = 'relay disconnected';
|
|
if (!reasons.includes('solver relay disconnected')) {
|
|
reasons.push('solver relay disconnected');
|
|
}
|
|
}
|
|
|
|
if (service === 'trade-executor') {
|
|
const saltState = state.verifier_salt_cache || health.verifier_salt_cache || null;
|
|
const queueDelay = state.executor_queue_delay || health.executor_queue_delay || null;
|
|
|
|
if (saltState?.configured && saltState.salt_fresh !== true) {
|
|
status = escalateHealth(status, 'warning');
|
|
if (label === 'healthy') label = 'salt unavailable';
|
|
reasons.push('verifier salt cache is stale or unavailable');
|
|
}
|
|
if (saltState?.last_refresh_error) {
|
|
status = escalateHealth(status, 'warning');
|
|
if (label === 'healthy') label = 'salt refresh failed';
|
|
reasons.push('verifier salt refresh failed');
|
|
}
|
|
if (queueDelay?.status === 'critical') {
|
|
status = escalateHealth(status, 'critical');
|
|
label = 'queue delayed';
|
|
reasons.push(queueDelay.warning_reason || 'executor queue delay is critical');
|
|
} else if (queueDelay?.status === 'warning') {
|
|
status = escalateHealth(status, 'warning');
|
|
if (label === 'healthy') label = 'queue delayed';
|
|
reasons.push(queueDelay.warning_reason || 'executor queue delay is high');
|
|
}
|
|
}
|
|
|
|
if (service === 'history-writer') {
|
|
if (state.database_connectivity === false) {
|
|
status = escalateHealth(status, 'critical');
|
|
label = 'database disconnected';
|
|
reasons.push('database connectivity failed');
|
|
} else if (freshnessAgeMs != null && freshnessAgeMs > 45_000) {
|
|
status = escalateHealth(status, 'warning');
|
|
label = 'writer lagging';
|
|
reasons.push('writer freshness degraded');
|
|
}
|
|
|
|
if (state.retention_mode === 'blocked_invalid_policy') {
|
|
status = escalateHealth(status, 'warning');
|
|
label = 'retention blocked';
|
|
reasons.push('quote lifecycle retention policy is invalid or missing');
|
|
} else if (state.retention_mode === 'blocked_rollup_stale') {
|
|
status = escalateHealth(status, 'warning');
|
|
label = 'rollup stale';
|
|
reasons.push('quote lifecycle rollup is stale, pruning is fail-closed');
|
|
} else if (state.retention_mode === 'pressure') {
|
|
status = escalateHealth(status, 'warning');
|
|
label = 'storage pressure';
|
|
reasons.push('quote lifecycle storage pressure retention mode is active');
|
|
}
|
|
}
|
|
|
|
if (service === 'operator-dashboard') {
|
|
if ((state.source_error_count || 0) > 0 || (health.source_error_count || 0) > 0) {
|
|
status = escalateHealth(status, 'warning');
|
|
label = 'sources degraded';
|
|
reasons.push('dashboard source degraded');
|
|
}
|
|
}
|
|
|
|
if (
|
|
['strategy-engine', 'trade-executor'].includes(service)
|
|
&& (state.armed ?? false)
|
|
&& hasCriticalTruthAlert(activeAlerts, activePairs.length ? activePairs : activePair)
|
|
) {
|
|
status = escalateHealth(status, 'critical');
|
|
if (label === 'healthy' || label === 'warning') {
|
|
label = 'armed on stale truth';
|
|
}
|
|
reasons.push('armed while critical upstream truth is stale');
|
|
}
|
|
|
|
return {
|
|
service,
|
|
status,
|
|
label,
|
|
reachable,
|
|
paused,
|
|
armed: state.armed ?? null,
|
|
health_ok: status === 'healthy' || status === 'paused',
|
|
highest_alert_severity: highestAlertSeverity,
|
|
reasons: dedupeReasons(reasons),
|
|
freshness_at: freshnessAt,
|
|
freshness_age_ms: freshnessAgeMs,
|
|
};
|
|
}
|
|
|
|
function classifyNearIntentsIngestHealth({
|
|
state,
|
|
health,
|
|
activeAlerts,
|
|
reachable,
|
|
now,
|
|
}) {
|
|
const reasons = [];
|
|
const alertCodes = new Set((activeAlerts || []).map((alert) => alert.alert_code));
|
|
const connected = state.ingest?.connected ?? health.connected ?? null;
|
|
const matchingAgeMs = ageMs(state.ingest?.last_matching_quote_at, now);
|
|
const publishedAgeMs = ageMs(state.ingest?.last_published_at, now);
|
|
|
|
if (!reachable) {
|
|
return {
|
|
status: 'offline',
|
|
label: 'offline',
|
|
reasons: ['service unreachable'],
|
|
};
|
|
}
|
|
|
|
if (connected === false || alertCodes.has('near_intents_ingest_disconnected')) {
|
|
return {
|
|
status: 'critical',
|
|
label: 'disconnected',
|
|
reasons: ['websocket disconnected', 'critical alert active (near_intents_ingest_disconnected)'],
|
|
};
|
|
}
|
|
|
|
if (
|
|
alertCodes.has('near_intents_publish_stale')
|
|
|| (
|
|
state.ingest?.last_matching_quote_at
|
|
&& state.ingest?.last_published_at
|
|
&& matchingAgeMs != null
|
|
&& publishedAgeMs != null
|
|
&& publishedAgeMs > matchingAgeMs + 5_000
|
|
)
|
|
) {
|
|
return {
|
|
status: 'critical',
|
|
label: 'publish stalled',
|
|
reasons: ['quote publish path stalled', 'critical alert active (near_intents_publish_stale)'],
|
|
};
|
|
}
|
|
|
|
if (alertCodes.has('near_intents_quotes_stale') || health.reason === 'quote truth stale') {
|
|
reasons.push('connected, no recent quotes for active pair');
|
|
if (matchingAgeMs != null) {
|
|
reasons.push(`last matching quote ${matchingAgeMs}ms ago`);
|
|
}
|
|
return {
|
|
status: 'warning',
|
|
label: 'no recent quotes',
|
|
reasons,
|
|
};
|
|
}
|
|
|
|
return {
|
|
status: health.ok === false ? 'warning' : 'healthy',
|
|
label: connected === true ? 'healthy' : 'unknown',
|
|
reasons: health.ok === false ? [health.reason || 'service health degraded'] : [],
|
|
};
|
|
}
|
|
|
|
function dedupeReasons(reasons) {
|
|
return [...new Set((reasons || []).filter(Boolean))];
|
|
}
|
|
|
|
export function inferServiceFreshnessTimestamp(service, state = {}, health = {}) {
|
|
switch (service) {
|
|
case 'near-intents-ingest':
|
|
return (
|
|
state.ingest?.last_published_at
|
|
|| state.ingest?.last_matching_quote_at
|
|
|| state.ingest?.last_message_at
|
|
|| null
|
|
);
|
|
case 'market-reference-ingest':
|
|
return state.last_published_at || null;
|
|
case 'inventory-sync':
|
|
return state.last_sync_at || null;
|
|
case 'liquidity-manager':
|
|
return state.last_refresh_at || null;
|
|
case 'history-writer':
|
|
return state.last_write_at || state.last_metrics_at || null;
|
|
case 'ops-sentinel':
|
|
return state.last_runtime_eval_at || state.last_evaluated_at || health.last_event_at || null;
|
|
case 'strategy-engine':
|
|
return state.latest_decision?.decision_at || state.latest_inventory_event?.ingested_at || null;
|
|
case 'trade-executor':
|
|
return state.relay?.last_message_at || state.last_quote_status?.created_at || null;
|
|
case 'operator-dashboard':
|
|
return state.last_bootstrap_at || state.last_source_error_at || null;
|
|
default:
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function buildRuntimeAlert({
|
|
alert_code,
|
|
severity,
|
|
reason,
|
|
service_scope,
|
|
pair = null,
|
|
asset_id = null,
|
|
tx_hash = null,
|
|
details = {},
|
|
}) {
|
|
return {
|
|
alert_code,
|
|
severity,
|
|
reason,
|
|
service_scope,
|
|
pair,
|
|
asset_id,
|
|
tx_hash,
|
|
details,
|
|
};
|
|
}
|
|
|
|
export function buildMakerCompetitivenessRuntimeAlerts({
|
|
makerCompetitiveness,
|
|
minPairSamples = 5,
|
|
minQuoteNotFoundOrFinishedRate = 0.4,
|
|
} = {}) {
|
|
const groups = Array.isArray(makerCompetitiveness?.groups) ? makerCompetitiveness.groups : [];
|
|
const pairStats = new Map();
|
|
|
|
for (const group of groups) {
|
|
const pair = group.pair || null;
|
|
if (!pair) continue;
|
|
const current = pairStats.get(pair) || {
|
|
pair,
|
|
sample_count: 0,
|
|
quote_not_found_or_finished_count: 0,
|
|
accepted_count: 0,
|
|
policy_skip_count: 0,
|
|
worst_group: null,
|
|
};
|
|
const count = Number(group.count || 0);
|
|
const staleFinishedCount = group.failure_category === 'quote_not_found_or_finished'
|
|
? count
|
|
: 0;
|
|
current.sample_count += count;
|
|
current.quote_not_found_or_finished_count += staleFinishedCount;
|
|
current.accepted_count += Number(group.accepted_count || 0);
|
|
current.policy_skip_count += Number(group.policy_skip_count || 0);
|
|
if (
|
|
staleFinishedCount > 0
|
|
&& (!current.worst_group || staleFinishedCount > Number(current.worst_group.count || 0))
|
|
) {
|
|
current.worst_group = group;
|
|
}
|
|
pairStats.set(pair, current);
|
|
}
|
|
|
|
return [...pairStats.values()]
|
|
.filter((stats) => {
|
|
if (stats.sample_count < minPairSamples) return false;
|
|
if (stats.quote_not_found_or_finished_count <= 0) return false;
|
|
return stats.quote_not_found_or_finished_count / stats.sample_count >= minQuoteNotFoundOrFinishedRate;
|
|
})
|
|
.map((stats) => {
|
|
const rate = stats.quote_not_found_or_finished_count / stats.sample_count;
|
|
return buildRuntimeAlert({
|
|
alert_code: 'maker_quote_not_found_or_finished_rate_high',
|
|
severity: 'warning',
|
|
reason: `maker responses for ${stats.pair} are frequently reaching relay after the quote is finished`,
|
|
service_scope: 'strategy-engine',
|
|
pair: stats.pair,
|
|
details: {
|
|
generated_at: makerCompetitiveness?.generated_at || null,
|
|
sample_count: stats.sample_count,
|
|
quote_not_found_or_finished_count: stats.quote_not_found_or_finished_count,
|
|
quote_not_found_or_finished_rate: Number(rate.toFixed(4)),
|
|
accepted_count: stats.accepted_count,
|
|
policy_skip_count: stats.policy_skip_count,
|
|
min_pair_samples: minPairSamples,
|
|
min_quote_not_found_or_finished_rate: minQuoteNotFoundOrFinishedRate,
|
|
worst_direction: stats.worst_group?.direction || null,
|
|
worst_request_kind: stats.worst_group?.request_kind || null,
|
|
worst_quote_age_bucket: stats.worst_group?.quote_age_bucket || null,
|
|
worst_notional_bucket: stats.worst_group?.notional_bucket || null,
|
|
},
|
|
});
|
|
});
|
|
}
|
|
|
|
export function buildExecutorRuntimeAlerts({
|
|
executor,
|
|
thresholds = createRuntimeHealthThresholds(),
|
|
now = new Date().toISOString(),
|
|
pair = null,
|
|
} = {}) {
|
|
void now;
|
|
const alerts = [];
|
|
const state = executor?.state || {};
|
|
const health = executor?.health || {};
|
|
const saltState = state.verifier_salt_cache || health.verifier_salt_cache || null;
|
|
const queueDelay = state.executor_queue_delay || health.executor_queue_delay || null;
|
|
|
|
if (saltState?.configured && saltState.salt_fresh !== true) {
|
|
alerts.push(buildRuntimeAlert({
|
|
alert_code: 'trade_executor_verifier_salt_unavailable',
|
|
severity: 'warning',
|
|
reason: saltState.has_cached_salt
|
|
? 'trade-executor verifier salt cache is stale'
|
|
: 'trade-executor verifier salt cache is unavailable',
|
|
service_scope: 'trade-executor',
|
|
pair,
|
|
details: {
|
|
has_cached_salt: saltState.has_cached_salt ?? null,
|
|
salt_fresh: saltState.salt_fresh ?? saltState.fresh ?? null,
|
|
salt_age_ms: saltState.salt_age_ms ?? saltState.cached_age_ms ?? null,
|
|
max_age_ms: saltState.max_age_ms ?? null,
|
|
refresh_in_flight: saltState.refresh_in_flight ?? null,
|
|
last_refresh_started_at: saltState.last_refresh_started_at || null,
|
|
last_refresh_completed_at: saltState.last_refresh_completed_at || null,
|
|
last_refresh_error: saltState.last_refresh_error || null,
|
|
},
|
|
}));
|
|
}
|
|
|
|
if (saltState?.last_refresh_error) {
|
|
alerts.push(buildRuntimeAlert({
|
|
alert_code: 'trade_executor_verifier_salt_refresh_failed',
|
|
severity: 'warning',
|
|
reason: 'trade-executor verifier salt refresh failed',
|
|
service_scope: 'trade-executor',
|
|
pair,
|
|
details: {
|
|
last_refresh_error: saltState.last_refresh_error,
|
|
last_refresh_started_at: saltState.last_refresh_started_at || null,
|
|
last_refresh_completed_at: saltState.last_refresh_completed_at || null,
|
|
last_refresh_duration_ms: saltState.last_refresh_duration_ms ?? null,
|
|
refresh_in_flight: saltState.refresh_in_flight ?? null,
|
|
has_cached_salt: saltState.has_cached_salt ?? null,
|
|
salt_fresh: saltState.salt_fresh ?? saltState.fresh ?? null,
|
|
},
|
|
}));
|
|
}
|
|
|
|
const queueMaxMs = Number(queueDelay?.max_ms);
|
|
if (Number.isFinite(queueMaxMs) && queueMaxMs >= thresholds.executorQueueDelayWarningMs) {
|
|
const critical = queueMaxMs >= thresholds.executorQueueDelayCriticalMs;
|
|
alerts.push(buildRuntimeAlert({
|
|
alert_code: 'trade_executor_command_queue_delay_high',
|
|
severity: critical ? 'critical' : 'warning',
|
|
reason: `trade-executor command queue delay ${queueMaxMs}ms exceeds ${
|
|
critical ? thresholds.executorQueueDelayCriticalMs : thresholds.executorQueueDelayWarningMs
|
|
}ms`,
|
|
service_scope: 'trade-executor',
|
|
pair,
|
|
details: {
|
|
status: queueDelay.status || (critical ? 'critical' : 'warning'),
|
|
latest_ms: queueDelay.latest_ms ?? null,
|
|
max_ms: queueDelay.max_ms ?? null,
|
|
p50_ms: queueDelay.p50_ms ?? null,
|
|
p90_ms: queueDelay.p90_ms ?? null,
|
|
p99_ms: queueDelay.p99_ms ?? null,
|
|
sample_count: queueDelay.sample_count ?? null,
|
|
warning_after_ms: thresholds.executorQueueDelayWarningMs,
|
|
critical_after_ms: thresholds.executorQueueDelayCriticalMs,
|
|
latest_command_id: queueDelay.latest_command_id || null,
|
|
latest_quote_id: queueDelay.latest_quote_id || null,
|
|
},
|
|
}));
|
|
}
|
|
|
|
return alerts;
|
|
}
|
|
|
|
export function shouldRaiseIngestPublishStale({
|
|
lastMatchingQuoteAt = null,
|
|
lastPublishedAt = null,
|
|
matchingQuoteAgeMs = null,
|
|
publishedAgeMs = null,
|
|
publishStaleMs,
|
|
} = {}) {
|
|
if (!lastMatchingQuoteAt) return false;
|
|
|
|
if (!lastPublishedAt) return true;
|
|
|
|
if (publishedAgeMs == null || publishedAgeMs > publishStaleMs) return true;
|
|
|
|
if (
|
|
matchingQuoteAgeMs != null
|
|
&& matchingQuoteAgeMs <= publishStaleMs
|
|
&& publishedAgeMs > matchingQuoteAgeMs + 5_000
|
|
) {
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
export function shouldContainExecutorForAlerts(alerts = []) {
|
|
void alerts;
|
|
return false;
|
|
}
|
|
|
|
export function ageMs(value, now = new Date().toISOString()) {
|
|
if (!value) return null;
|
|
const left = new Date(value).getTime();
|
|
const right = new Date(now).getTime();
|
|
if (!Number.isFinite(left) || !Number.isFinite(right)) return null;
|
|
return Math.max(0, right - left);
|
|
}
|
|
|
|
function highestAlertSeverityForService(alerts) {
|
|
let highest = null;
|
|
let highestRank = -1;
|
|
for (const alert of alerts || []) {
|
|
const rank = ALERT_RANK[alert.severity] ?? -1;
|
|
if (rank > highestRank) {
|
|
highest = alert.severity;
|
|
highestRank = rank;
|
|
}
|
|
}
|
|
return highest;
|
|
}
|
|
|
|
function indexAlertsByService(activeAlerts) {
|
|
const index = new Map();
|
|
for (const alert of activeAlerts || []) {
|
|
const list = index.get(alert.service_scope) || [];
|
|
list.push(alert);
|
|
index.set(alert.service_scope, list);
|
|
}
|
|
return index;
|
|
}
|
|
|
|
function hasCriticalTruthAlert(alerts, activePair) {
|
|
const activePairSet = new Set(Array.isArray(activePair) ? activePair : [activePair].filter(Boolean));
|
|
return (alerts || []).some((alert) => (
|
|
alert.severity === 'critical'
|
|
&& (
|
|
alert.pair == null
|
|
|| activePairSet.has(alert.pair)
|
|
|| alert.alert_code.includes('stale')
|
|
|| alert.alert_code.includes('disconnected')
|
|
)
|
|
));
|
|
}
|
|
|
|
function escalateHealth(current, next) {
|
|
const currentRank = HEALTH_RANK[current] ?? 0;
|
|
const nextRank = HEALTH_RANK[next] ?? 0;
|
|
return nextRank > currentRank ? next : current;
|
|
}
|