import { createHash } from 'node:crypto'; import { normalizeCompetitivenessEntry, notionalBucket, quoteAgeBucket, } from './maker-competitiveness.mjs'; import { timestampMs, toIsoTimestamp } from './maker-timing.mjs'; import { TERMINAL_SETTLEMENT_ATTRIBUTION_STATUSES } from './quote-outcomes.mjs'; export const QUOTE_LIFECYCLE_RETENTION_POLICY_KEY = 'quote_lifecycle_default'; export const QUOTE_LIFECYCLE_DETAIL_TABLES = [ 'raw_near_intents_quotes', 'swap_demand_events', 'trade_decisions', 'execute_trade_commands', 'trade_execution_results', 'quote_outcome_attributions', ]; export const QUOTE_LIFECYCLE_DATA_CLASSES = Object.freeze({ successfulTradeEvidence: 'successful_trade_evidence', inFlightDetail: 'in_flight_detail', nonSuccessDetail: 'non_success_detail', rawDebugFirehose: 'raw_debug_firehose', aggregateRollup: 'aggregate_rollup', }); export const SUCCESSFUL_SETTLEMENT_ATTRIBUTION_STATUSES = new Set([ ...TERMINAL_SETTLEMENT_ATTRIBUTION_STATUSES, 'exact_match', 'attributed', ]); export const DEFAULT_QUOTE_LIFECYCLE_RETENTION_POLICY = Object.freeze({ policy_key: QUOTE_LIFECYCLE_RETENTION_POLICY_KEY, enabled: true, normal_detail_retention_ms: 30 * 60 * 1000, pressure_detail_retention_ms: 10 * 60 * 1000, raw_retention_ms: 30 * 60 * 1000, rollup_granularity_ms: 5 * 60 * 1000, rollup_retention_ms: 90 * 24 * 60 * 60 * 1000, rollup_stale_after_ms: 15 * 60 * 1000, preserve_successful_evidence: true, max_delete_rows_per_pass: 2_000_000, pressure_table_bytes: 1_500_000_000, pressure_database_bytes: 3_500_000_000, config_version: 2, }); export function isSuccessfulQuoteOutcome(row = {}) { const payload = row.payload || row.outcome || {}; const outcomeStatus = normalizeToken(row.outcome_status || payload.outcome_status); const attributionStatus = normalizeToken(row.attribution_status || payload.attribution_status); return outcomeStatus === 'completed' || SUCCESSFUL_SETTLEMENT_ATTRIBUTION_STATUSES.has(attributionStatus); } export function classifyQuoteLifecycleDataClass({ table = null, row = {}, now = new Date().toISOString(), detailRetentionMs = DEFAULT_QUOTE_LIFECYCLE_RETENTION_POLICY.normal_detail_retention_ms, } = {}) { if (table === 'quote_lifecycle_rollups' || row.data_class === QUOTE_LIFECYCLE_DATA_CLASSES.aggregateRollup) { return QUOTE_LIFECYCLE_DATA_CLASSES.aggregateRollup; } if (isSuccessfulQuoteOutcome(row)) { return QUOTE_LIFECYCLE_DATA_CLASSES.successfulTradeEvidence; } if (table === 'raw_near_intents_quotes' && !resolveQuoteId(row)) { return QUOTE_LIFECYCLE_DATA_CLASSES.rawDebugFirehose; } const nowMs = timestampMs(now); const rowMs = timestampMs( row.outcome_observed_at || row.computed_at || row.observed_at || row.ingested_at || row.payload?.observed_at || row.payload?.ingested_at, ); if ( Number.isFinite(nowMs) && Number.isFinite(rowMs) && Number.isInteger(detailRetentionMs) && nowMs - rowMs <= detailRetentionMs ) { return QUOTE_LIFECYCLE_DATA_CLASSES.inFlightDetail; } return QUOTE_LIFECYCLE_DATA_CLASSES.nonSuccessDetail; } export function normalizeQuoteLifecycleRetentionPolicy(row = null) { if (!row) { return { ok: false, block_reason: 'retention_policy_missing', policy: null, }; } const policy = { policy_key: String(row.policy_key || '').trim(), enabled: row.enabled === true, normal_detail_retention_ms: positiveInteger(row.normal_detail_retention_ms), pressure_detail_retention_ms: positiveInteger(row.pressure_detail_retention_ms), raw_retention_ms: positiveInteger(row.raw_retention_ms), rollup_granularity_ms: positiveInteger(row.rollup_granularity_ms), rollup_retention_ms: positiveInteger(row.rollup_retention_ms), rollup_stale_after_ms: positiveInteger(row.rollup_stale_after_ms), preserve_successful_evidence: row.preserve_successful_evidence === true, max_delete_rows_per_pass: positiveInteger(row.max_delete_rows_per_pass), pressure_table_bytes: positiveInteger(row.pressure_table_bytes), pressure_database_bytes: positiveInteger(row.pressure_database_bytes), updated_at: toIsoTimestamp(row.updated_at), config_version: positiveInteger(row.config_version), }; const invalidFields = Object.entries(policy) .filter(([field, value]) => ( [ 'normal_detail_retention_ms', 'pressure_detail_retention_ms', 'raw_retention_ms', 'rollup_granularity_ms', 'rollup_retention_ms', 'rollup_stale_after_ms', 'max_delete_rows_per_pass', 'pressure_table_bytes', 'pressure_database_bytes', 'config_version', ].includes(field) && value == null )) .map(([field]) => field); if (!policy.policy_key) invalidFields.push('policy_key'); if (policy.enabled !== true) invalidFields.push('enabled'); if (policy.preserve_successful_evidence !== true) invalidFields.push('preserve_successful_evidence'); if ( policy.pressure_detail_retention_ms != null && policy.normal_detail_retention_ms != null && policy.pressure_detail_retention_ms > policy.normal_detail_retention_ms ) { invalidFields.push('pressure_detail_retention_ms'); } if (invalidFields.length) { return { ok: false, block_reason: `retention_policy_invalid:${invalidFields.join(',')}`, policy: null, }; } return { ok: true, block_reason: null, policy, }; } export function decideQuoteLifecycleRetentionMode({ policy, storageMetrics = null, } = {}) { const normalized = policy?.policy ? policy : normalizeQuoteLifecycleRetentionPolicy(policy); if (!normalized.ok) { return { mode: 'blocked_invalid_policy', pressure: false, reason: normalized.block_reason, detail_retention_ms: null, }; } const activePolicy = normalized.policy; const pressureTables = (storageMetrics?.tables || []) .filter((table) => Number(table.total_bytes || 0) >= activePolicy.pressure_table_bytes) .map((table) => table.table); const databasePressure = Number(storageMetrics?.database?.total_bytes || 0) >= activePolicy.pressure_database_bytes; const pressure = pressureTables.length > 0 || databasePressure; return { mode: pressure ? 'pressure' : 'normal', pressure, reason: pressure ? [ pressureTables.length ? `tables:${pressureTables.join(',')}` : null, databasePressure ? 'database_bytes' : null, ].filter(Boolean).join(';') : null, detail_retention_ms: pressure ? activePolicy.pressure_detail_retention_ms : activePolicy.normal_detail_retention_ms, }; } export function buildQuoteLifecycleRollups({ rows = [], rollupGranularityMs = DEFAULT_QUOTE_LIFECYCLE_RETENTION_POLICY.rollup_granularity_ms, computedAt = new Date().toISOString(), now = computedAt, } = {}) { const groups = new Map(); for (const row of rows || []) { const normalized = normalizeRollupInput({ ...row, now }); const activityAt = normalized.activity_at; const windowStart = floorTimestamp(activityAt, rollupGranularityMs); if (!windowStart) continue; const windowEnd = new Date(Date.parse(windowStart) + rollupGranularityMs).toISOString(); const timing = normalized.entry.maker_timing || {}; const sourceDataClass = normalized.source_data_class; const keyParts = [ windowStart, windowEnd, sourceDataClass, normalized.entry.pair || 'unknown', normalized.entry.direction || 'unknown', normalized.entry.request_kind || 'unknown', normalized.entry.edge_bps ?? 'unknown', normalized.notional_asset || 'unknown', normalized.entry.notional_bucket || 'unavailable', normalized.entry.quote_age_bucket || 'unavailable', normalized.entry.result_code || 'no_result', normalized.entry.failure_category || 'none', normalized.entry.outcome_status || 'unknown', ]; const key = JSON.stringify(keyParts); const existing = groups.get(key) || { rollup_id: buildRollupId(keyParts), window_start: windowStart, window_end: windowEnd, data_class: QUOTE_LIFECYCLE_DATA_CLASSES.aggregateRollup, source_data_class: sourceDataClass, pair: normalized.entry.pair || null, direction: normalized.entry.direction || 'unknown', request_kind: normalized.entry.request_kind || 'unknown', edge_bps: normalized.entry.edge_bps == null ? null : String(normalized.entry.edge_bps), notional_asset: normalized.notional_asset, notional_bucket: normalized.entry.notional_bucket || notionalBucket(null, normalized.notional_symbol), quote_age_bucket: normalized.entry.quote_age_bucket || quoteAgeBucket(null), result_code: normalized.entry.result_code || 'no_result', failure_category: normalized.entry.failure_category || null, outcome_status: normalized.entry.outcome_status || 'unknown', quote_count: 0, decision_count: 0, command_count: 0, executor_result_count: 0, relay_accepted_count: 0, relay_failed_count: 0, not_filled_count: 0, completed_count: 0, timing_bucket_counts: {}, min_observed_at: normalized.activity_at, max_observed_at: normalized.activity_at, source_watermark_at: normalized.activity_at, computed_at: computedAt, }; existing.quote_count += 1; existing.decision_count += normalized.has_decision ? 1 : 0; existing.command_count += normalized.has_command ? 1 : 0; existing.executor_result_count += normalized.has_execution ? 1 : 0; existing.relay_accepted_count += normalized.entry.accepted ? 1 : 0; existing.relay_failed_count += normalized.entry.relay_failed ? 1 : 0; existing.not_filled_count += normalized.entry.outcome_status === 'not_filled' ? 1 : 0; existing.completed_count += isSuccessfulQuoteOutcome(normalized.outcome || { outcome_status: normalized.entry.outcome_status }) ? 1 : 0; existing.min_observed_at = earlierTimestamp(existing.min_observed_at, normalized.activity_at); existing.max_observed_at = laterTimestamp(existing.max_observed_at, normalized.activity_at); existing.source_watermark_at = laterTimestamp(existing.source_watermark_at, normalized.activity_at); addTimingBuckets(existing.timing_bucket_counts, timing); groups.set(key, existing); } return [...groups.values()].sort((left, right) => ( String(left.window_start).localeCompare(String(right.window_start)) || String(left.pair || '').localeCompare(String(right.pair || '')) || String(left.result_code || '').localeCompare(String(right.result_code || '')) )); } export function normalizeQuoteLifecycleRollupRow(row = {}) { return { rollup_id: row.rollup_id, window_start: toIsoTimestamp(row.window_start), window_end: toIsoTimestamp(row.window_end), data_class: row.data_class || QUOTE_LIFECYCLE_DATA_CLASSES.aggregateRollup, source_data_class: row.source_data_class || null, pair: row.pair || null, direction: row.direction || 'unknown', request_kind: row.request_kind || 'unknown', edge_bps: row.edge_bps == null ? null : String(row.edge_bps), notional_asset: row.notional_asset || null, notional_bucket: row.notional_bucket || 'unavailable', quote_age_bucket: row.quote_age_bucket || 'unavailable', result_code: row.result_code || 'no_result', failure_category: row.failure_category || null, outcome_status: row.outcome_status || 'unknown', quote_count: Number(row.quote_count || 0), decision_count: Number(row.decision_count || 0), command_count: Number(row.command_count || 0), executor_result_count: Number(row.executor_result_count || 0), relay_accepted_count: Number(row.relay_accepted_count || 0), relay_failed_count: Number(row.relay_failed_count || 0), not_filled_count: Number(row.not_filled_count || 0), completed_count: Number(row.completed_count || 0), timing_bucket_counts: row.timing_bucket_counts || {}, min_observed_at: toIsoTimestamp(row.min_observed_at), max_observed_at: toIsoTimestamp(row.max_observed_at), source_watermark_at: toIsoTimestamp(row.source_watermark_at), computed_at: toIsoTimestamp(row.computed_at), }; } function normalizeRollupInput(row = {}) { const decision = row.decision || row.decision_payload || null; const command = row.command || row.command_payload || null; const execution = row.execution || row.execution_payload || null; const outcome = row.outcome || row.outcome_payload || null; const quote = row.quote || row.quote_payload || row.demand || row.demand_payload || {}; const makerTiming = outcome?.maker_timing || execution?.maker_timing || command?.maker_timing || decision?.maker_timing || quote?.maker_timing || row.maker_timing || {}; const entry = normalizeCompetitivenessEntry({ quote_id: row.quote_id || quote.quote_id || decision?.quote_id || command?.quote_id || execution?.quote_id || outcome?.quote_id || null, pair: row.pair || quote.pair || decision?.pair || command?.pair || execution?.pair || outcome?.pair || null, pair_id: row.pair_id || decision?.pair_id || command?.pair_id || execution?.pair_id || null, direction: row.direction || decision?.direction || command?.direction || outcome?.direction || null, request_kind: row.request_kind || quote.request_kind || decision?.request_kind || command?.request_kind || outcome?.request_kind || null, edge_bps: row.edge_bps ?? decision?.edge_bps ?? command?.edge_bps ?? outcome?.edge_bps ?? null, notional: row.notional ?? decision?.notional ?? command?.notional ?? outcome?.notional ?? quote?.notional ?? null, notional_symbol: row.notional_symbol || decision?.notional_symbol || command?.notional_symbol || outcome?.notional_symbol || null, maker_timing: makerTiming, lifecycle_state: row.lifecycle_state || null, outcome_status: outcome?.outcome_status || row.outcome_status || execution?.outcome_status || null, execution, decision, command, }); const sourceTable = row.source_table || row.table || null; const activityAt = firstTimestamp([ row.activity_at, outcome?.outcome_observed_at, row.outcome_observed_at, execution?.result_at, row.execution_result_at, command?.command_at, row.command_at, decision?.decision_at, row.decision_at, quote?.observed_at, row.quote_observed_at, row.observed_at, row.ingested_at, row.computed_at, ]); const sourceDataClass = classifyQuoteLifecycleDataClass({ table: sourceTable, row: { ...row, payload: outcome || execution || decision || quote || {}, outcome_status: outcome?.outcome_status || row.outcome_status, attribution_status: outcome?.attribution_status || row.attribution_status, observed_at: activityAt, }, now: row.now || new Date().toISOString(), }); return { entry, outcome, source_data_class: sourceDataClass, source_table: sourceTable, notional_asset: row.notional_asset || row.notional_asset_id || decision?.notional_asset_id || command?.notional_asset_id || outcome?.notional_asset_id || null, notional_symbol: row.notional_symbol || decision?.notional_symbol || command?.notional_symbol || outcome?.notional_symbol || null, activity_at: activityAt, has_decision: Boolean(decision?.decision_id || decision?.decision || row.decision_id), has_command: Boolean(command?.command_id || row.command_id), has_execution: Boolean(execution?.command_id || execution?.status || row.execution_result_status), }; } function addTimingBuckets(target, timing = {}) { for (const [field, value] of Object.entries(timing || {})) { if (!field.endsWith('_ms')) continue; const bucket = fixedTimingBucket(value); target[field] ||= {}; target[field][bucket] = Number(target[field][bucket] || 0) + 1; } } function fixedTimingBucket(value) { const number = Number(value); if (!Number.isFinite(number)) return 'unavailable'; if (number < 50) return '<50ms'; if (number < 100) return '50-100ms'; if (number < 250) return '100-250ms'; if (number < 500) return '250-500ms'; if (number < 1000) return '500-1000ms'; if (number < 5000) return '1000-5000ms'; return '>=5000ms'; } function buildRollupId(parts) { return `quote-lifecycle-rollup:${createHash('sha256').update(JSON.stringify(parts)).digest('hex').slice(0, 32)}`; } function resolveQuoteId(row = {}) { return row.quote_id || row.payload?.quote_id || row.outcome?.quote_id || null; } function positiveInteger(value) { const number = Number(value); return Number.isInteger(number) && number > 0 ? number : null; } function normalizeToken(value) { return String(value || '').trim().toLowerCase(); } function firstTimestamp(values = []) { for (const value of values) { const iso = toIsoTimestamp(value); if (iso) return iso; } return null; } function floorTimestamp(value, granularityMs) { const ms = timestampMs(value); if (!Number.isFinite(ms)) return null; const granularity = positiveInteger(granularityMs); if (!granularity) return null; return new Date(Math.floor(ms / granularity) * granularity).toISOString(); } function earlierTimestamp(left, right) { const leftMs = timestampMs(left); const rightMs = timestampMs(right); if (!Number.isFinite(leftMs)) return toIsoTimestamp(right); if (!Number.isFinite(rightMs)) return toIsoTimestamp(left); return new Date(Math.min(leftMs, rightMs)).toISOString(); } function laterTimestamp(left, right) { const leftMs = timestampMs(left); const rightMs = timestampMs(right); if (!Number.isFinite(leftMs)) return toIsoTimestamp(right); if (!Number.isFinite(rightMs)) return toIsoTimestamp(left); return new Date(Math.max(leftMs, rightMs)).toISOString(); }