2d329c9b2f
All checks were successful
deploy / deploy (push) Successful in 57s
Proof: DB-backed quote lifecycle retention policy, rollup-before-prune maintenance, dashboard storage visibility, and regression tests for preservation and fail-closed behavior. Assumptions: Completed quote outcome attribution remains the current durable successful-trade evidence; venue-native terminal fill ids and fee-complete PnL remain unavailable. Still fake: Historical non-success detail already pruned cannot be recovered; rollups are aggregate analytics, not per-quote settled trade proof.
492 lines
18 KiB
JavaScript
492 lines
18 KiB
JavaScript
import { createHash } from 'node:crypto';
|
|
|
|
import {
|
|
normalizeCompetitivenessEntry,
|
|
notionalBucket,
|
|
quoteAgeBucket,
|
|
} from './maker-competitiveness.mjs';
|
|
import { TERMINAL_SETTLEMENT_ATTRIBUTION_STATUSES } from './quote-outcomes.mjs';
|
|
|
|
export const QUOTE_LIFECYCLE_RETENTION_POLICY_KEY = 'quote_lifecycle_default';
|
|
|
|
export const QUOTE_LIFECYCLE_DETAIL_TABLES = [
|
|
'raw_near_intents_quotes',
|
|
'swap_demand_events',
|
|
'trade_decisions',
|
|
'execute_trade_commands',
|
|
'trade_execution_results',
|
|
'quote_outcome_attributions',
|
|
];
|
|
|
|
export const QUOTE_LIFECYCLE_DATA_CLASSES = Object.freeze({
|
|
successfulTradeEvidence: 'successful_trade_evidence',
|
|
inFlightDetail: 'in_flight_detail',
|
|
nonSuccessDetail: 'non_success_detail',
|
|
rawDebugFirehose: 'raw_debug_firehose',
|
|
aggregateRollup: 'aggregate_rollup',
|
|
});
|
|
|
|
export const SUCCESSFUL_SETTLEMENT_ATTRIBUTION_STATUSES = new Set([
|
|
...TERMINAL_SETTLEMENT_ATTRIBUTION_STATUSES,
|
|
'exact_match',
|
|
'attributed',
|
|
]);
|
|
|
|
export const DEFAULT_QUOTE_LIFECYCLE_RETENTION_POLICY = Object.freeze({
|
|
policy_key: QUOTE_LIFECYCLE_RETENTION_POLICY_KEY,
|
|
enabled: true,
|
|
normal_detail_retention_ms: 30 * 60 * 1000,
|
|
pressure_detail_retention_ms: 10 * 60 * 1000,
|
|
raw_retention_ms: 30 * 60 * 1000,
|
|
rollup_granularity_ms: 5 * 60 * 1000,
|
|
rollup_retention_ms: 90 * 24 * 60 * 60 * 1000,
|
|
rollup_stale_after_ms: 15 * 60 * 1000,
|
|
preserve_successful_evidence: true,
|
|
max_delete_rows_per_pass: 100_000,
|
|
pressure_table_bytes: 1_500_000_000,
|
|
pressure_database_bytes: 3_500_000_000,
|
|
config_version: 1,
|
|
});
|
|
|
|
export function isSuccessfulQuoteOutcome(row = {}) {
|
|
const payload = row.payload || row.outcome || {};
|
|
const outcomeStatus = normalizeToken(row.outcome_status || payload.outcome_status);
|
|
const attributionStatus = normalizeToken(row.attribution_status || payload.attribution_status);
|
|
return outcomeStatus === 'completed'
|
|
|| SUCCESSFUL_SETTLEMENT_ATTRIBUTION_STATUSES.has(attributionStatus);
|
|
}
|
|
|
|
export function classifyQuoteLifecycleDataClass({
|
|
table = null,
|
|
row = {},
|
|
now = new Date().toISOString(),
|
|
detailRetentionMs = DEFAULT_QUOTE_LIFECYCLE_RETENTION_POLICY.normal_detail_retention_ms,
|
|
} = {}) {
|
|
if (table === 'quote_lifecycle_rollups' || row.data_class === QUOTE_LIFECYCLE_DATA_CLASSES.aggregateRollup) {
|
|
return QUOTE_LIFECYCLE_DATA_CLASSES.aggregateRollup;
|
|
}
|
|
|
|
if (isSuccessfulQuoteOutcome(row)) {
|
|
return QUOTE_LIFECYCLE_DATA_CLASSES.successfulTradeEvidence;
|
|
}
|
|
|
|
if (table === 'raw_near_intents_quotes' && !resolveQuoteId(row)) {
|
|
return QUOTE_LIFECYCLE_DATA_CLASSES.rawDebugFirehose;
|
|
}
|
|
|
|
const nowMs = timestampMs(now);
|
|
const rowMs = timestampMs(
|
|
row.outcome_observed_at
|
|
|| row.computed_at
|
|
|| row.observed_at
|
|
|| row.ingested_at
|
|
|| row.payload?.observed_at
|
|
|| row.payload?.ingested_at,
|
|
);
|
|
if (
|
|
Number.isFinite(nowMs)
|
|
&& Number.isFinite(rowMs)
|
|
&& Number.isInteger(detailRetentionMs)
|
|
&& nowMs - rowMs <= detailRetentionMs
|
|
) {
|
|
return QUOTE_LIFECYCLE_DATA_CLASSES.inFlightDetail;
|
|
}
|
|
|
|
return QUOTE_LIFECYCLE_DATA_CLASSES.nonSuccessDetail;
|
|
}
|
|
|
|
export function normalizeQuoteLifecycleRetentionPolicy(row = null) {
|
|
if (!row) {
|
|
return {
|
|
ok: false,
|
|
block_reason: 'retention_policy_missing',
|
|
policy: null,
|
|
};
|
|
}
|
|
|
|
const policy = {
|
|
policy_key: String(row.policy_key || '').trim(),
|
|
enabled: row.enabled === true,
|
|
normal_detail_retention_ms: positiveInteger(row.normal_detail_retention_ms),
|
|
pressure_detail_retention_ms: positiveInteger(row.pressure_detail_retention_ms),
|
|
raw_retention_ms: positiveInteger(row.raw_retention_ms),
|
|
rollup_granularity_ms: positiveInteger(row.rollup_granularity_ms),
|
|
rollup_retention_ms: positiveInteger(row.rollup_retention_ms),
|
|
rollup_stale_after_ms: positiveInteger(row.rollup_stale_after_ms),
|
|
preserve_successful_evidence: row.preserve_successful_evidence === true,
|
|
max_delete_rows_per_pass: positiveInteger(row.max_delete_rows_per_pass),
|
|
pressure_table_bytes: positiveInteger(row.pressure_table_bytes),
|
|
pressure_database_bytes: positiveInteger(row.pressure_database_bytes),
|
|
updated_at: toIsoTimestamp(row.updated_at),
|
|
config_version: positiveInteger(row.config_version),
|
|
};
|
|
|
|
const invalidFields = Object.entries(policy)
|
|
.filter(([field, value]) => (
|
|
[
|
|
'normal_detail_retention_ms',
|
|
'pressure_detail_retention_ms',
|
|
'raw_retention_ms',
|
|
'rollup_granularity_ms',
|
|
'rollup_retention_ms',
|
|
'rollup_stale_after_ms',
|
|
'max_delete_rows_per_pass',
|
|
'pressure_table_bytes',
|
|
'pressure_database_bytes',
|
|
'config_version',
|
|
].includes(field)
|
|
&& value == null
|
|
))
|
|
.map(([field]) => field);
|
|
|
|
if (!policy.policy_key) invalidFields.push('policy_key');
|
|
if (policy.enabled !== true) invalidFields.push('enabled');
|
|
if (policy.preserve_successful_evidence !== true) invalidFields.push('preserve_successful_evidence');
|
|
if (
|
|
policy.pressure_detail_retention_ms != null
|
|
&& policy.normal_detail_retention_ms != null
|
|
&& policy.pressure_detail_retention_ms > policy.normal_detail_retention_ms
|
|
) {
|
|
invalidFields.push('pressure_detail_retention_ms');
|
|
}
|
|
|
|
if (invalidFields.length) {
|
|
return {
|
|
ok: false,
|
|
block_reason: `retention_policy_invalid:${invalidFields.join(',')}`,
|
|
policy: null,
|
|
};
|
|
}
|
|
|
|
return {
|
|
ok: true,
|
|
block_reason: null,
|
|
policy,
|
|
};
|
|
}
|
|
|
|
export function decideQuoteLifecycleRetentionMode({
|
|
policy,
|
|
storageMetrics = null,
|
|
} = {}) {
|
|
const normalized = policy?.policy ? policy : normalizeQuoteLifecycleRetentionPolicy(policy);
|
|
if (!normalized.ok) {
|
|
return {
|
|
mode: 'blocked_invalid_policy',
|
|
pressure: false,
|
|
reason: normalized.block_reason,
|
|
detail_retention_ms: null,
|
|
};
|
|
}
|
|
|
|
const activePolicy = normalized.policy;
|
|
const pressureTables = (storageMetrics?.tables || [])
|
|
.filter((table) => Number(table.total_bytes || 0) >= activePolicy.pressure_table_bytes)
|
|
.map((table) => table.table);
|
|
const databasePressure = Number(storageMetrics?.database?.total_bytes || 0) >= activePolicy.pressure_database_bytes;
|
|
const pressure = pressureTables.length > 0 || databasePressure;
|
|
|
|
return {
|
|
mode: pressure ? 'pressure' : 'normal',
|
|
pressure,
|
|
reason: pressure
|
|
? [
|
|
pressureTables.length ? `tables:${pressureTables.join(',')}` : null,
|
|
databasePressure ? 'database_bytes' : null,
|
|
].filter(Boolean).join(';')
|
|
: null,
|
|
detail_retention_ms: pressure
|
|
? activePolicy.pressure_detail_retention_ms
|
|
: activePolicy.normal_detail_retention_ms,
|
|
};
|
|
}
|
|
|
|
export function buildQuoteLifecycleRollups({
|
|
rows = [],
|
|
rollupGranularityMs = DEFAULT_QUOTE_LIFECYCLE_RETENTION_POLICY.rollup_granularity_ms,
|
|
computedAt = new Date().toISOString(),
|
|
now = computedAt,
|
|
} = {}) {
|
|
const groups = new Map();
|
|
for (const row of rows || []) {
|
|
const normalized = normalizeRollupInput({ ...row, now });
|
|
const activityAt = normalized.activity_at;
|
|
const windowStart = floorTimestamp(activityAt, rollupGranularityMs);
|
|
if (!windowStart) continue;
|
|
const windowEnd = new Date(Date.parse(windowStart) + rollupGranularityMs).toISOString();
|
|
const timing = normalized.entry.maker_timing || {};
|
|
const sourceDataClass = normalized.source_data_class;
|
|
const keyParts = [
|
|
windowStart,
|
|
windowEnd,
|
|
sourceDataClass,
|
|
normalized.entry.pair || 'unknown',
|
|
normalized.entry.direction || 'unknown',
|
|
normalized.entry.request_kind || 'unknown',
|
|
normalized.entry.edge_bps ?? 'unknown',
|
|
normalized.notional_asset || 'unknown',
|
|
normalized.entry.notional_bucket || 'unavailable',
|
|
normalized.entry.quote_age_bucket || 'unavailable',
|
|
normalized.entry.result_code || 'no_result',
|
|
normalized.entry.failure_category || 'none',
|
|
normalized.entry.outcome_status || 'unknown',
|
|
];
|
|
const key = JSON.stringify(keyParts);
|
|
const existing = groups.get(key) || {
|
|
rollup_id: buildRollupId(keyParts),
|
|
window_start: windowStart,
|
|
window_end: windowEnd,
|
|
data_class: QUOTE_LIFECYCLE_DATA_CLASSES.aggregateRollup,
|
|
source_data_class: sourceDataClass,
|
|
pair: normalized.entry.pair || null,
|
|
direction: normalized.entry.direction || 'unknown',
|
|
request_kind: normalized.entry.request_kind || 'unknown',
|
|
edge_bps: normalized.entry.edge_bps == null ? null : String(normalized.entry.edge_bps),
|
|
notional_asset: normalized.notional_asset,
|
|
notional_bucket: normalized.entry.notional_bucket || notionalBucket(null, normalized.notional_symbol),
|
|
quote_age_bucket: normalized.entry.quote_age_bucket || quoteAgeBucket(null),
|
|
result_code: normalized.entry.result_code || 'no_result',
|
|
failure_category: normalized.entry.failure_category || null,
|
|
outcome_status: normalized.entry.outcome_status || 'unknown',
|
|
quote_count: 0,
|
|
decision_count: 0,
|
|
command_count: 0,
|
|
executor_result_count: 0,
|
|
relay_accepted_count: 0,
|
|
relay_failed_count: 0,
|
|
not_filled_count: 0,
|
|
completed_count: 0,
|
|
timing_bucket_counts: {},
|
|
min_observed_at: normalized.activity_at,
|
|
max_observed_at: normalized.activity_at,
|
|
source_watermark_at: normalized.activity_at,
|
|
computed_at: computedAt,
|
|
};
|
|
|
|
existing.quote_count += 1;
|
|
existing.decision_count += normalized.has_decision ? 1 : 0;
|
|
existing.command_count += normalized.has_command ? 1 : 0;
|
|
existing.executor_result_count += normalized.has_execution ? 1 : 0;
|
|
existing.relay_accepted_count += normalized.entry.accepted ? 1 : 0;
|
|
existing.relay_failed_count += normalized.entry.relay_failed ? 1 : 0;
|
|
existing.not_filled_count += normalized.entry.outcome_status === 'not_filled' ? 1 : 0;
|
|
existing.completed_count += isSuccessfulQuoteOutcome(normalized.outcome || { outcome_status: normalized.entry.outcome_status })
|
|
? 1
|
|
: 0;
|
|
existing.min_observed_at = earlierTimestamp(existing.min_observed_at, normalized.activity_at);
|
|
existing.max_observed_at = laterTimestamp(existing.max_observed_at, normalized.activity_at);
|
|
existing.source_watermark_at = laterTimestamp(existing.source_watermark_at, normalized.activity_at);
|
|
addTimingBuckets(existing.timing_bucket_counts, timing);
|
|
groups.set(key, existing);
|
|
}
|
|
|
|
return [...groups.values()].sort((left, right) => (
|
|
String(left.window_start).localeCompare(String(right.window_start))
|
|
|| String(left.pair || '').localeCompare(String(right.pair || ''))
|
|
|| String(left.result_code || '').localeCompare(String(right.result_code || ''))
|
|
));
|
|
}
|
|
|
|
export function normalizeQuoteLifecycleRollupRow(row = {}) {
|
|
return {
|
|
rollup_id: row.rollup_id,
|
|
window_start: toIsoTimestamp(row.window_start),
|
|
window_end: toIsoTimestamp(row.window_end),
|
|
data_class: row.data_class || QUOTE_LIFECYCLE_DATA_CLASSES.aggregateRollup,
|
|
source_data_class: row.source_data_class || null,
|
|
pair: row.pair || null,
|
|
direction: row.direction || 'unknown',
|
|
request_kind: row.request_kind || 'unknown',
|
|
edge_bps: row.edge_bps == null ? null : String(row.edge_bps),
|
|
notional_asset: row.notional_asset || null,
|
|
notional_bucket: row.notional_bucket || 'unavailable',
|
|
quote_age_bucket: row.quote_age_bucket || 'unavailable',
|
|
result_code: row.result_code || 'no_result',
|
|
failure_category: row.failure_category || null,
|
|
outcome_status: row.outcome_status || 'unknown',
|
|
quote_count: Number(row.quote_count || 0),
|
|
decision_count: Number(row.decision_count || 0),
|
|
command_count: Number(row.command_count || 0),
|
|
executor_result_count: Number(row.executor_result_count || 0),
|
|
relay_accepted_count: Number(row.relay_accepted_count || 0),
|
|
relay_failed_count: Number(row.relay_failed_count || 0),
|
|
not_filled_count: Number(row.not_filled_count || 0),
|
|
completed_count: Number(row.completed_count || 0),
|
|
timing_bucket_counts: row.timing_bucket_counts || {},
|
|
min_observed_at: toIsoTimestamp(row.min_observed_at),
|
|
max_observed_at: toIsoTimestamp(row.max_observed_at),
|
|
source_watermark_at: toIsoTimestamp(row.source_watermark_at),
|
|
computed_at: toIsoTimestamp(row.computed_at),
|
|
};
|
|
}
|
|
|
|
function normalizeRollupInput(row = {}) {
|
|
const decision = row.decision || row.decision_payload || null;
|
|
const command = row.command || row.command_payload || null;
|
|
const execution = row.execution || row.execution_payload || null;
|
|
const outcome = row.outcome || row.outcome_payload || null;
|
|
const quote = row.quote || row.quote_payload || row.demand || row.demand_payload || {};
|
|
const makerTiming =
|
|
outcome?.maker_timing
|
|
|| execution?.maker_timing
|
|
|| command?.maker_timing
|
|
|| decision?.maker_timing
|
|
|| quote?.maker_timing
|
|
|| row.maker_timing
|
|
|| {};
|
|
const entry = normalizeCompetitivenessEntry({
|
|
quote_id: row.quote_id || quote.quote_id || decision?.quote_id || command?.quote_id || execution?.quote_id || outcome?.quote_id || null,
|
|
pair: row.pair || quote.pair || decision?.pair || command?.pair || execution?.pair || outcome?.pair || null,
|
|
pair_id: row.pair_id || decision?.pair_id || command?.pair_id || execution?.pair_id || null,
|
|
direction: row.direction || decision?.direction || command?.direction || outcome?.direction || null,
|
|
request_kind: row.request_kind || quote.request_kind || decision?.request_kind || command?.request_kind || outcome?.request_kind || null,
|
|
edge_bps: row.edge_bps ?? decision?.edge_bps ?? command?.edge_bps ?? outcome?.edge_bps ?? null,
|
|
notional: row.notional ?? decision?.notional ?? command?.notional ?? outcome?.notional ?? quote?.notional ?? null,
|
|
notional_symbol:
|
|
row.notional_symbol
|
|
|| decision?.notional_symbol
|
|
|| command?.notional_symbol
|
|
|| outcome?.notional_symbol
|
|
|| null,
|
|
maker_timing: makerTiming,
|
|
lifecycle_state: row.lifecycle_state || null,
|
|
outcome_status: outcome?.outcome_status || row.outcome_status || execution?.outcome_status || null,
|
|
execution,
|
|
decision,
|
|
command,
|
|
});
|
|
const sourceTable = row.source_table || row.table || null;
|
|
const activityAt = firstTimestamp([
|
|
row.activity_at,
|
|
outcome?.outcome_observed_at,
|
|
row.outcome_observed_at,
|
|
execution?.result_at,
|
|
row.execution_result_at,
|
|
command?.command_at,
|
|
row.command_at,
|
|
decision?.decision_at,
|
|
row.decision_at,
|
|
quote?.observed_at,
|
|
row.quote_observed_at,
|
|
row.observed_at,
|
|
row.ingested_at,
|
|
row.computed_at,
|
|
]);
|
|
const sourceDataClass = classifyQuoteLifecycleDataClass({
|
|
table: sourceTable,
|
|
row: {
|
|
...row,
|
|
payload: outcome || execution || decision || quote || {},
|
|
outcome_status: outcome?.outcome_status || row.outcome_status,
|
|
attribution_status: outcome?.attribution_status || row.attribution_status,
|
|
observed_at: activityAt,
|
|
},
|
|
now: row.now || new Date().toISOString(),
|
|
});
|
|
|
|
return {
|
|
entry,
|
|
outcome,
|
|
source_data_class: sourceDataClass,
|
|
source_table: sourceTable,
|
|
notional_asset:
|
|
row.notional_asset
|
|
|| row.notional_asset_id
|
|
|| decision?.notional_asset_id
|
|
|| command?.notional_asset_id
|
|
|| outcome?.notional_asset_id
|
|
|| null,
|
|
notional_symbol:
|
|
row.notional_symbol
|
|
|| decision?.notional_symbol
|
|
|| command?.notional_symbol
|
|
|| outcome?.notional_symbol
|
|
|| null,
|
|
activity_at: activityAt,
|
|
has_decision: Boolean(decision?.decision_id || decision?.decision || row.decision_id),
|
|
has_command: Boolean(command?.command_id || row.command_id),
|
|
has_execution: Boolean(execution?.command_id || execution?.status || row.execution_result_status),
|
|
};
|
|
}
|
|
|
|
function addTimingBuckets(target, timing = {}) {
|
|
for (const [field, value] of Object.entries(timing || {})) {
|
|
if (!field.endsWith('_ms')) continue;
|
|
const bucket = fixedTimingBucket(value);
|
|
target[field] ||= {};
|
|
target[field][bucket] = Number(target[field][bucket] || 0) + 1;
|
|
}
|
|
}
|
|
|
|
function fixedTimingBucket(value) {
|
|
const number = Number(value);
|
|
if (!Number.isFinite(number)) return 'unavailable';
|
|
if (number < 50) return '<50ms';
|
|
if (number < 100) return '50-100ms';
|
|
if (number < 250) return '100-250ms';
|
|
if (number < 500) return '250-500ms';
|
|
if (number < 1000) return '500-1000ms';
|
|
if (number < 5000) return '1000-5000ms';
|
|
return '>=5000ms';
|
|
}
|
|
|
|
function buildRollupId(parts) {
|
|
return `quote-lifecycle-rollup:${createHash('sha256').update(JSON.stringify(parts)).digest('hex').slice(0, 32)}`;
|
|
}
|
|
|
|
function resolveQuoteId(row = {}) {
|
|
return row.quote_id || row.payload?.quote_id || row.outcome?.quote_id || null;
|
|
}
|
|
|
|
function positiveInteger(value) {
|
|
const number = Number(value);
|
|
return Number.isInteger(number) && number > 0 ? number : null;
|
|
}
|
|
|
|
function normalizeToken(value) {
|
|
return String(value || '').trim().toLowerCase();
|
|
}
|
|
|
|
function timestampMs(value) {
|
|
if (!value) return NaN;
|
|
const parsed = Date.parse(value);
|
|
return Number.isFinite(parsed) ? parsed : NaN;
|
|
}
|
|
|
|
function toIsoTimestamp(value) {
|
|
if (!value) return null;
|
|
const date = new Date(value);
|
|
return Number.isNaN(date.getTime()) ? null : date.toISOString();
|
|
}
|
|
|
|
function firstTimestamp(values = []) {
|
|
for (const value of values) {
|
|
const iso = toIsoTimestamp(value);
|
|
if (iso) return iso;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function floorTimestamp(value, granularityMs) {
|
|
const ms = timestampMs(value);
|
|
if (!Number.isFinite(ms)) return null;
|
|
const granularity = positiveInteger(granularityMs);
|
|
if (!granularity) return null;
|
|
return new Date(Math.floor(ms / granularity) * granularity).toISOString();
|
|
}
|
|
|
|
function earlierTimestamp(left, right) {
|
|
const leftMs = timestampMs(left);
|
|
const rightMs = timestampMs(right);
|
|
if (!Number.isFinite(leftMs)) return toIsoTimestamp(right);
|
|
if (!Number.isFinite(rightMs)) return toIsoTimestamp(left);
|
|
return new Date(Math.min(leftMs, rightMs)).toISOString();
|
|
}
|
|
|
|
function laterTimestamp(left, right) {
|
|
const leftMs = timestampMs(left);
|
|
const rightMs = timestampMs(right);
|
|
if (!Number.isFinite(leftMs)) return toIsoTimestamp(right);
|
|
if (!Number.isFinite(rightMs)) return toIsoTimestamp(left);
|
|
return new Date(Math.max(leftMs, rightMs)).toISOString();
|
|
}
|