From 503a69e71ce79771395d6c43272b5494a1910904 Mon Sep 17 00:00:00 2001 From: philipp Date: Sat, 13 Jun 2026 14:04:20 +0200 Subject: [PATCH] Retain quote lifecycle statistics across pruning Proof: added persisted quote_lifecycle_stat_subjects snapshots, refreshed statistics from retained per-quote subjects, preserved completed/attributed execution evidence during lifecycle retention, and wired successful lifecycle evidence into the dashboard trade funnel; verified with targeted lifecycle/dashboard tests, full npm test, and operator dashboard bundle build. Assumptions: detailed successful trade rows already deleted by prior retention cannot be reconstructed from aggregate rollups without faking quote-level settlement evidence; the new subject table starts preserving surviving and future quote evidence after deployment. Still fake: fee-aware realized PnL and venue-native fill truth beyond retained settlement/inventory evidence remain out of scope; already-pruned detailed success rows remain unrecoverable. --- src/apps/operator-dashboard.mjs | 9 + src/core/operator-dashboard.mjs | 73 ++- src/core/quote-lifecycle-statistics.mjs | 3 + src/lib/postgres.mjs | 435 +++++++++++++++++- test/inventory-and-history.test.mjs | 2 + test/operator-dashboard.test.mjs | 96 ++++ ...stgres-quote-lifecycle-statistics.test.mjs | 139 ++++++ test/quote-lifecycle-statistics.test.mjs | 22 + 8 files changed, 763 insertions(+), 16 deletions(-) diff --git a/src/apps/operator-dashboard.mjs b/src/apps/operator-dashboard.mjs index 87e5ed0..55d8d7a 100644 --- a/src/apps/operator-dashboard.mjs +++ b/src/apps/operator-dashboard.mjs @@ -59,6 +59,7 @@ import { loadRecentQuoteOutcomes, loadRecentTradeDecisions, loadRecentQuotes, + loadSuccessfulQuoteLifecycleEvidence, loadSubmissionPage, loadSubmissionSummary, pauseTradingPair, @@ -495,6 +496,7 @@ async function loadBootstrapPayload({ auth, page, pageSize }) { recentExecuteTradeCommands, recentExecutionResults, recentQuoteOutcomes, + successfulQuoteLifecycle, quoteLifecycleRetention, quoteLifecycleRollups, quoteLifecycleStatistics, @@ -569,6 +571,12 @@ async function loadBootstrapPayload({ auth, page, pageSize }) { [], sourceErrors, ), + safeSourceLoad( + 'successful_quote_lifecycle', + () => loadSuccessfulQuoteLifecycleEvidence(pool, { limit: 50 }), + null, + sourceErrors, + ), safeSourceLoad( 'quote_lifecycle_retention', () => loadQuoteLifecycleRetentionSummary(pool), @@ -631,6 +639,7 @@ async function loadBootstrapPayload({ auth, page, pageSize }) { recentExecuteTradeCommands, recentExecutionResults, recentQuoteOutcomes, + successfulQuoteLifecycle, quoteLifecycleRetention, quoteLifecycleRollups, quoteLifecycleStatistics: { diff --git a/src/core/operator-dashboard.mjs b/src/core/operator-dashboard.mjs index 29227af..aec3da1 100644 --- a/src/core/operator-dashboard.mjs +++ b/src/core/operator-dashboard.mjs @@ -558,6 +558,7 @@ export function buildDashboardBootstrap({ recentExecuteTradeCommands, recentExecutionResults, recentQuoteOutcomes = [], + successfulQuoteLifecycle = null, quoteLifecycleRetention = null, quoteLifecycleRollups = [], quoteLifecycleStatistics = null, @@ -655,6 +656,7 @@ export function buildDashboardBootstrap({ recentExecuteTradeCommands, recentExecutionResults, recentQuoteOutcomes, + successfulQuoteLifecycle, quoteLifecycleRollups, quoteLifecycleStatistics, }), @@ -1751,6 +1753,7 @@ function buildStrategySummary({ recentExecuteTradeCommands = [], recentExecutionResults = [], recentQuoteOutcomes = [], + successfulQuoteLifecycle = null, quoteLifecycleRollups = [], quoteLifecycleStatistics = null, }) { @@ -1788,12 +1791,20 @@ function buildStrategySummary({ || durableDecisionsById.get(strategyState.latest_decision?.decision_id)?.decision_at || null, }); - const allLifecycleRows = deriveQuoteLifecycleRows({ + const lifecycleEvidence = mergeSuccessfulQuoteLifecycleEvidence({ recentQuotes, recentTradeDecisions, recentExecuteTradeCommands, recentExecutionResults, recentQuoteOutcomes, + successfulQuoteLifecycle, + }); + const allLifecycleRows = deriveQuoteLifecycleRows({ + recentQuotes: lifecycleEvidence.recentQuotes, + recentTradeDecisions: lifecycleEvidence.recentTradeDecisions, + recentExecuteTradeCommands: lifecycleEvidence.recentExecuteTradeCommands, + recentExecutionResults: lifecycleEvidence.recentExecutionResults, + recentQuoteOutcomes: lifecycleEvidence.recentQuoteOutcomes, limit: null, }).map((row) => enrichLifecycleRowForUi({ config, row })); const lifecycleRows = allLifecycleRows.slice(0, 20); @@ -2000,6 +2011,66 @@ function buildFallbackPairConfig(config) { }; } +function mergeSuccessfulQuoteLifecycleEvidence({ + recentQuotes = [], + recentTradeDecisions = [], + recentExecuteTradeCommands = [], + recentExecutionResults = [], + recentQuoteOutcomes = [], + successfulQuoteLifecycle = null, +} = {}) { + if (!successfulQuoteLifecycle?.found) { + return { + recentQuotes, + recentTradeDecisions, + recentExecuteTradeCommands, + recentExecutionResults, + recentQuoteOutcomes, + }; + } + + return { + recentQuotes: mergeLifecycleEvidenceRows( + recentQuotes, + successfulQuoteLifecycle.recentQuotes, + (row) => row?.quote_id, + ), + recentTradeDecisions: mergeLifecycleEvidenceRows( + recentTradeDecisions, + successfulQuoteLifecycle.recentTradeDecisions, + (row) => row?.payload?.decision_id || row?.payload?.quote_id || row?.quote_id, + ), + recentExecuteTradeCommands: mergeLifecycleEvidenceRows( + recentExecuteTradeCommands, + successfulQuoteLifecycle.recentExecuteTradeCommands, + (row) => row?.command_id || row?.decision_id || row?.quote_id, + ), + recentExecutionResults: mergeLifecycleEvidenceRows( + recentExecutionResults, + successfulQuoteLifecycle.recentExecutionResults, + (row) => row?.command_id || `${row?.quote_id || 'quote'}:${row?.result_at || row?.observed_at || ''}`, + ), + recentQuoteOutcomes: mergeLifecycleEvidenceRows( + recentQuoteOutcomes, + successfulQuoteLifecycle.recentQuoteOutcomes, + (row) => row?.quote_id || row?.command_id || row?.decision_id, + ), + }; +} + +function mergeLifecycleEvidenceRows(primary = [], additional = [], keyFn = null) { + const merged = []; + const seen = new Set(); + for (const row of [...(primary || []), ...(additional || [])]) { + if (!row) continue; + const key = keyFn?.(row) || JSON.stringify(row); + if (key && seen.has(key)) continue; + if (key) seen.add(key); + merged.push(row); + } + return merged; +} + function summarizeGrossEdgeEstimate(rows = []) { let total = 0n; let count = 0; diff --git a/src/core/quote-lifecycle-statistics.mjs b/src/core/quote-lifecycle-statistics.mjs index 3de2de5..a48d893 100644 --- a/src/core/quote-lifecycle-statistics.mjs +++ b/src/core/quote-lifecycle-statistics.mjs @@ -48,6 +48,9 @@ export function emptyQuoteLifecycleBucketCounts() { export function classifyQuoteLifecycleStatisticsBucket(row = null) { if (!row) return 'unavailable'; + const explicitBucket = normalizeToken(row.statistics_bucket || row.bucket); + if (QUOTE_LIFECYCLE_STATISTIC_BUCKETS.includes(explicitBucket)) return explicitBucket; + const lifecycleState = normalizeToken(row.lifecycle_state); const executionStatus = normalizeToken(row.execution?.status || row.stage_details?.execution_status); const outcomeStatus = normalizeToken( diff --git a/src/lib/postgres.mjs b/src/lib/postgres.mjs index 05f4890..a44a8fc 100644 --- a/src/lib/postgres.mjs +++ b/src/lib/postgres.mjs @@ -7,6 +7,7 @@ import { deriveQuoteOutcomeRecords } from '../core/quote-outcomes.mjs'; import { QUOTE_LIFECYCLE_STATISTIC_GRAINS, buildQuoteLifecycleStatistics, + classifyQuoteLifecycleStatisticsBucket, normalizeQuoteLifecycleStatisticsGrains, normalizeQuoteLifecycleStatisticsRow, } from '../core/quote-lifecycle-statistics.mjs'; @@ -61,6 +62,7 @@ const QUOTE_LIFECYCLE_ROLLUPS_TABLE = 'quote_lifecycle_rollups'; const QUOTE_LIFECYCLE_ROLLUP_WATERMARKS_TABLE = 'quote_lifecycle_rollup_watermarks'; const QUOTE_LIFECYCLE_RETENTION_RUNS_TABLE = 'quote_lifecycle_retention_runs'; const QUOTE_LIFECYCLE_STATISTICS_TABLE = 'quote_lifecycle_statistics'; +const QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE = 'quote_lifecycle_stat_subjects'; const QUOTE_LIFECYCLE_ROLLUP_WINDOWS_PER_PASS = 48; const SUPPORTED_ASSET_IMPORT_RUNS_TABLE = 'supported_asset_import_runs'; const TRADING_ASSETS_TABLE = 'trading_assets'; @@ -419,6 +421,27 @@ export async function ensureQuoteLifecycleRetentionSchema(pool) { ON ${QUOTE_LIFECYCLE_STATISTICS_TABLE} (computed_at DESC) `); + await pool.query(` + CREATE TABLE IF NOT EXISTS ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE} ( + quote_id TEXT PRIMARY KEY, + first_activity_at TIMESTAMPTZ, + latest_stage_at TIMESTAMPTZ, + bucket TEXT NOT NULL, + lifecycle_state TEXT, + reason_code TEXT, + updated_at TIMESTAMPTZ NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb + ) + `); + await pool.query(` + CREATE INDEX IF NOT EXISTS ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}_first_activity_idx + ON ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE} (first_activity_at ASC NULLS LAST) + `); + await pool.query(` + CREATE INDEX IF NOT EXISTS ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}_bucket_idx + ON ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE} (bucket, latest_stage_at DESC NULLS LAST) + `); + await pool.query(` CREATE TABLE IF NOT EXISTS ${QUOTE_LIFECYCLE_ROLLUP_WATERMARKS_TABLE} ( rollup_key TEXT PRIMARY KEY, @@ -3192,8 +3215,17 @@ export async function refreshQuoteLifecycleStatistics(pool, { recentQuoteOutcomes: evidence.recentQuoteOutcomes, limit: null, }); - const statistics = buildQuoteLifecycleStatistics({ + const subjectUpsertResult = await upsertQuoteLifecycleStatisticSubjects(pool, { lifecycleRows, + now, + }); + const statisticSubjectRows = await loadQuoteLifecycleStatisticSubjects(pool); + const missingIdentifierRows = lifecycleRows.filter((row) => !String(row?.quote_id || '').trim()); + const statistics = buildQuoteLifecycleStatistics({ + lifecycleRows: [ + ...statisticSubjectRows, + ...missingIdentifierRows, + ], grains, computedAt: now, }); @@ -3209,11 +3241,229 @@ export async function refreshQuoteLifecycleStatistics(pool, { execution_results: evidence.recentExecutionResults.length, outcomes: evidence.recentQuoteOutcomes.length, lifecycle_rows: lifecycleRows.length, + statistic_subjects: statisticSubjectRows.length, + statistic_subject_upserts: subjectUpsertResult.upserted_count, }, statistics, }; } +async function upsertQuoteLifecycleStatisticSubjects(pool, { + lifecycleRows = [], + now = new Date().toISOString(), +} = {}) { + let upsertedCount = 0; + for (const row of lifecycleRows || []) { + const subject = buildQuoteLifecycleStatisticSubject(row, { now }); + if (!subject) continue; + await pool.query( + ` + INSERT INTO ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE} ( + quote_id, + first_activity_at, + latest_stage_at, + bucket, + lifecycle_state, + reason_code, + updated_at, + payload + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8::jsonb) + ON CONFLICT (quote_id) DO UPDATE SET + first_activity_at = CASE + WHEN ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.first_activity_at IS NULL THEN EXCLUDED.first_activity_at + WHEN EXCLUDED.first_activity_at IS NULL THEN ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.first_activity_at + ELSE LEAST(${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.first_activity_at, EXCLUDED.first_activity_at) + END, + latest_stage_at = CASE + WHEN ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.latest_stage_at IS NULL THEN EXCLUDED.latest_stage_at + WHEN EXCLUDED.latest_stage_at IS NULL THEN ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.latest_stage_at + ELSE GREATEST(${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.latest_stage_at, EXCLUDED.latest_stage_at) + END, + bucket = CASE + WHEN ${quoteLifecycleBucketRankSql('EXCLUDED.bucket')} > ${quoteLifecycleBucketRankSql(`${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.bucket`)} + OR ( + ${quoteLifecycleBucketRankSql('EXCLUDED.bucket')} = ${quoteLifecycleBucketRankSql(`${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.bucket`)} + AND COALESCE(EXCLUDED.latest_stage_at, '-infinity'::timestamptz) >= COALESCE(${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.latest_stage_at, '-infinity'::timestamptz) + ) + THEN EXCLUDED.bucket + ELSE ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.bucket + END, + lifecycle_state = CASE + WHEN ${quoteLifecycleBucketRankSql('EXCLUDED.bucket')} > ${quoteLifecycleBucketRankSql(`${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.bucket`)} + OR ( + ${quoteLifecycleBucketRankSql('EXCLUDED.bucket')} = ${quoteLifecycleBucketRankSql(`${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.bucket`)} + AND COALESCE(EXCLUDED.latest_stage_at, '-infinity'::timestamptz) >= COALESCE(${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.latest_stage_at, '-infinity'::timestamptz) + ) + THEN EXCLUDED.lifecycle_state + ELSE ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.lifecycle_state + END, + reason_code = CASE + WHEN ${quoteLifecycleBucketRankSql('EXCLUDED.bucket')} > ${quoteLifecycleBucketRankSql(`${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.bucket`)} + OR ( + ${quoteLifecycleBucketRankSql('EXCLUDED.bucket')} = ${quoteLifecycleBucketRankSql(`${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.bucket`)} + AND COALESCE(EXCLUDED.latest_stage_at, '-infinity'::timestamptz) >= COALESCE(${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.latest_stage_at, '-infinity'::timestamptz) + ) + THEN EXCLUDED.reason_code + ELSE ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.reason_code + END, + payload = CASE + WHEN ${quoteLifecycleBucketRankSql('EXCLUDED.bucket')} > ${quoteLifecycleBucketRankSql(`${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.bucket`)} + OR ( + ${quoteLifecycleBucketRankSql('EXCLUDED.bucket')} = ${quoteLifecycleBucketRankSql(`${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.bucket`)} + AND COALESCE(EXCLUDED.latest_stage_at, '-infinity'::timestamptz) >= COALESCE(${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.latest_stage_at, '-infinity'::timestamptz) + ) + THEN EXCLUDED.payload + ELSE ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.payload + END, + updated_at = GREATEST(${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE}.updated_at, EXCLUDED.updated_at) + `, + [ + subject.quote_id, + subject.first_activity_at, + subject.latest_stage_at, + subject.bucket, + subject.lifecycle_state, + subject.reason_code, + subject.updated_at, + JSON.stringify(subject.payload), + ], + ); + upsertedCount += 1; + } + return { upserted_count: upsertedCount }; +} + +async function loadQuoteLifecycleStatisticSubjects(pool) { + const result = await pool.query( + ` + SELECT + quote_id, + first_activity_at, + latest_stage_at, + bucket, + lifecycle_state, + reason_code, + updated_at, + payload + FROM ${QUOTE_LIFECYCLE_STATISTIC_SUBJECTS_TABLE} + ORDER BY first_activity_at ASC NULLS LAST, quote_id ASC + `, + ); + return result.rows.map(normalizeQuoteLifecycleStatisticSubjectRow); +} + +function buildQuoteLifecycleStatisticSubject(row = {}, { now = new Date().toISOString() } = {}) { + const quoteId = String(row?.quote_id || '').trim(); + if (!quoteId) return null; + const firstActivityAt = firstQuoteLifecycleStatisticTimestamp(row); + const latestStageAt = latestQuoteLifecycleStatisticTimestamp(row); + const bucket = classifyQuoteLifecycleStatisticsBucket(row); + return { + quote_id: quoteId, + first_activity_at: firstActivityAt, + latest_stage_at: latestStageAt, + bucket, + lifecycle_state: row.lifecycle_state || null, + reason_code: row.reason_code || null, + updated_at: toIsoTimestamp(now) || new Date().toISOString(), + payload: { + quote_id: quoteId, + decision_id: row.decision_id || null, + command_id: row.command_id || null, + lifecycle_state: row.lifecycle_state || null, + lifecycle_label: row.lifecycle_label || null, + reason_code: row.reason_code || null, + statistics_bucket: bucket, + first_activity_at: firstActivityAt, + latest_stage_at: latestStageAt, + outcome_status: row.outcome_status || row.outcome?.outcome_status || null, + attribution_status: row.attribution_status || row.outcome?.attribution_status || null, + has_settlement_evidence: row.has_settlement_evidence === true, + }, + }; +} + +function normalizeQuoteLifecycleStatisticSubjectRow(row = {}) { + const payload = row.payload || {}; + return { + quote_id: row.quote_id || payload.quote_id || null, + decision_id: payload.decision_id || null, + command_id: payload.command_id || null, + quote_activity_at: toIsoTimestamp(row.first_activity_at || payload.first_activity_at), + latest_stage_at: toIsoTimestamp(row.latest_stage_at || payload.latest_stage_at), + lifecycle_state: row.lifecycle_state || payload.lifecycle_state || lifecycleStateForStatisticBucket(row.bucket), + reason_code: row.reason_code || payload.reason_code || row.bucket || 'unavailable', + statistics_bucket: row.bucket || payload.statistics_bucket || 'unavailable', + }; +} + +function firstQuoteLifecycleStatisticTimestamp(row = {}) { + return earliestTimestamp([ + row.quote_activity_at, + row.quote_observed_at, + row.decision_at, + row.command_at, + row.execution_result_at, + row.outcome_observed_at, + row.latest_stage_at, + ]); +} + +function latestQuoteLifecycleStatisticTimestamp(row = {}) { + return latestTimestamp([ + row.latest_stage_at, + row.outcome_observed_at, + row.execution_result_at, + row.command_at, + row.decision_at, + row.quote_observed_at, + row.quote_activity_at, + ]); +} + +function lifecycleStateForStatisticBucket(bucket) { + switch (bucket) { + case 'success': + return 'completed'; + case 'not_filled': + return 'not_filled'; + case 'submission_failed': + return 'failed'; + case 'submitted_no_reply': + return 'submitted'; + case 'awaiting_executor': + return 'command_emitted'; + case 'approved_no_command': + return 'evaluated'; + case 'blocked_before_submit': + return 'blocked'; + case 'rejected_by_strategy': + return 'rejected'; + case 'observed_no_decision': + return 'observed'; + case 'unavailable': + default: + return 'observed'; + } +} + +function quoteLifecycleBucketRankSql(expression) { + return `( + CASE ${expression} + WHEN 'success' THEN 8 + WHEN 'not_filled' THEN 7 + WHEN 'submission_failed' THEN 6 + WHEN 'submitted_no_reply' THEN 5 + WHEN 'rejected_by_strategy' THEN 4 + WHEN 'blocked_before_submit' THEN 4 + WHEN 'awaiting_executor' THEN 3 + WHEN 'approved_no_command' THEN 2 + WHEN 'observed_no_decision' THEN 1 + ELSE 0 + END + )`; +} + export async function loadQuoteLifecycleStatistics(pool, { grains = QUOTE_LIFECYCLE_STATISTIC_GRAINS, limit = 96, @@ -3426,13 +3676,24 @@ function quoteLifecyclePruneTableSpecs() { } function successfulQuoteEvidencePredicate(spec) { - if (spec.selfOutcome) return successfulQuoteEvidenceTargetPredicate(); + if (spec.selfOutcome) { + return ` + ( + ${successfulQuoteEvidenceTargetPredicate()} + OR ${successfulExecutionEvidenceExistsPredicate('target')} + ) + `; + } return ` - EXISTS ( - SELECT 1 - FROM quote_outcome_attributions outcome - WHERE outcome.quote_id = target.quote_id - AND ${successfulQuoteEvidenceSelfPredicate('outcome')} + ( + ${successfulQuotePayloadPredicate('target.payload')} + OR ${successfulExecutionEvidenceExistsPredicate('target')} + OR EXISTS ( + SELECT 1 + FROM quote_outcome_attributions outcome + WHERE outcome.quote_id = target.quote_id + AND ${successfulQuoteEvidenceSelfPredicate('outcome')} + ) ) `; } @@ -3447,6 +3708,26 @@ function successfulQuoteEvidenceSelfPredicate(alias = null) { `; } +function successfulQuotePayloadPredicate(payloadExpression) { + return ` + ( + ${payloadExpression}->>'outcome_status' = 'completed' + OR ${payloadExpression}->>'attribution_status' IN ('heuristic_match', 'linked_settlement', 'exact_match', 'attributed') + ) + `; +} + +function successfulExecutionEvidenceExistsPredicate(targetExpression) { + return ` + EXISTS ( + SELECT 1 + FROM trade_execution_results execution + WHERE execution.quote_id = ${targetExpression}.quote_id + AND ${successfulQuotePayloadPredicate('execution.payload')} + ) + `; +} + function successfulQuoteEvidenceTargetPredicate() { return ` ( @@ -3555,17 +3836,39 @@ export async function pruneNonSuccessfulQuoteLifecycleHistory(pool, { ( target.outcome_status = 'completed' OR target.attribution_status IN ('heuristic_match', 'linked_settlement', 'exact_match', 'attributed') + OR EXISTS ( + SELECT 1 + FROM trade_execution_results execution + WHERE execution.quote_id = target.quote_id + AND ( + execution.payload->>'outcome_status' = 'completed' + OR execution.payload->>'attribution_status' IN ('heuristic_match', 'linked_settlement', 'exact_match', 'attributed') + ) + ) ) ` : ` - EXISTS ( - SELECT 1 - FROM quote_outcome_attributions outcome - WHERE outcome.quote_id = target.quote_id - AND ( - outcome.outcome_status = 'completed' - OR outcome.attribution_status IN ('heuristic_match', 'linked_settlement', 'exact_match', 'attributed') - ) + ( + target.payload->>'outcome_status' = 'completed' + OR target.payload->>'attribution_status' IN ('heuristic_match', 'linked_settlement', 'exact_match', 'attributed') + OR EXISTS ( + SELECT 1 + FROM trade_execution_results execution + WHERE execution.quote_id = target.quote_id + AND ( + execution.payload->>'outcome_status' = 'completed' + OR execution.payload->>'attribution_status' IN ('heuristic_match', 'linked_settlement', 'exact_match', 'attributed') + ) + ) + OR EXISTS ( + SELECT 1 + FROM quote_outcome_attributions outcome + WHERE outcome.quote_id = target.quote_id + AND ( + outcome.outcome_status = 'completed' + OR outcome.attribution_status IN ('heuristic_match', 'linked_settlement', 'exact_match', 'attributed') + ) + ) ) `; const result = await pool.query( @@ -4085,6 +4388,88 @@ export async function loadRecentQuoteOutcomes(pool, { limit = 200 } = {}) { return result.rows.map(normalizeQuoteOutcomeRow); } +export async function loadSuccessfulQuoteLifecycleEvidence(pool, { limit = 50 } = {}) { + const lookupAt = new Date().toISOString(); + const boundedLimit = Math.max(1, Math.min(500, Number(limit) || 50)); + const candidatesResult = await pool.query( + ` + WITH successful_quotes AS ( + SELECT + quote_id, + COALESCE(outcome_observed_at, submitted_at, command_at, computed_at) AS matched_at + FROM ${QUOTE_OUTCOMES_TABLE} + WHERE quote_id IS NOT NULL + AND ( + outcome_status = 'completed' + OR attribution_status IN ('heuristic_match', 'linked_settlement', 'exact_match', 'attributed') + ) + + UNION ALL + + SELECT + quote_id, + COALESCE(observed_at, ingested_at) AS matched_at + FROM trade_execution_results + WHERE quote_id IS NOT NULL + AND ( + payload->>'outcome_status' = 'completed' + OR payload->>'attribution_status' IN ('heuristic_match', 'linked_settlement', 'exact_match', 'attributed') + ) + ), ranked AS ( + SELECT + quote_id, + MAX(matched_at) AS matched_at + FROM successful_quotes + GROUP BY quote_id + ) + SELECT quote_id, matched_at + FROM ranked + ORDER BY matched_at DESC NULLS LAST + LIMIT $1 + `, + [boundedLimit], + ); + + const identifiers = { + quote_ids: uniqueTextValues(candidatesResult.rows.map((row) => row.quote_id)), + decision_ids: [], + command_ids: [], + }; + if (!hasLookupIdentifiers(identifiers)) { + return buildQuoteLifecycleLookupEvidence({ + requestedIdentifier: 'successful_quote_lifecycle', + lookupAt, + matchedIdentifiers: identifiers, + }); + } + + const [ + recentQuotes, + recentTradeDecisions, + recentExecuteTradeCommands, + recentExecutionResults, + recentQuoteOutcomes, + ] = await Promise.all([ + loadLifecycleLookupQuotes(pool, identifiers), + loadLifecycleLookupTradeDecisions(pool, identifiers), + loadLifecycleLookupExecuteTradeCommands(pool, identifiers), + loadLifecycleLookupExecutionResults(pool, identifiers), + loadLifecycleLookupQuoteOutcomes(pool, identifiers), + ]); + + return buildQuoteLifecycleLookupEvidence({ + requestedIdentifier: 'successful_quote_lifecycle', + lookupAt, + matchedIdentifierType: 'quote_id', + matchedIdentifiers: identifiers, + recentQuotes, + recentTradeDecisions, + recentExecuteTradeCommands, + recentExecutionResults, + recentQuoteOutcomes, + }); +} + export async function loadQuoteLifecycleLookupEvidence(pool, { identifier } = {}) { const requestedIdentifier = String(identifier || '').trim(); const lookupAt = new Date().toISOString(); @@ -5686,6 +6071,26 @@ function floorTimestamp(value, granularityMs) { return new Date(Math.floor(parsed / granularity) * granularity).toISOString(); } +function earliestTimestamp(values = []) { + let earliest = null; + for (const value of values || []) { + const iso = toIsoTimestamp(value); + if (!iso) continue; + if (!earliest || timestampValue(iso) < timestampValue(earliest)) earliest = iso; + } + return earliest; +} + +function latestTimestamp(values = []) { + let latest = null; + for (const value of values || []) { + const iso = toIsoTimestamp(value); + if (!iso) continue; + if (!latest || timestampValue(iso) > timestampValue(latest)) latest = iso; + } + return latest; +} + function laterTimestamp(left, right) { const leftMs = timestampValue(left); const rightMs = timestampValue(right); diff --git a/test/inventory-and-history.test.mjs b/test/inventory-and-history.test.mjs index 78599aa..35aec23 100644 --- a/test/inventory-and-history.test.mjs +++ b/test/inventory-and-history.test.mjs @@ -194,6 +194,8 @@ test('quote lifecycle retention preserves completed trades and drains old non-su for (const query of queries) { assert.match(query.sql, /DELETE FROM/); assert.match(query.sql, /outcome_status = 'completed'/); + assert.match(query.sql, /payload->>'outcome_status' = 'completed'/); + assert.match(query.sql, /FROM trade_execution_results execution/); assert.match(query.sql, /attribution_status IN \('heuristic_match', 'linked_settlement', 'exact_match', 'attributed'\)/); assert.deepEqual(query.params, ['2026-06-05T11:30:00.000Z', 100_000]); } diff --git a/test/operator-dashboard.test.mjs b/test/operator-dashboard.test.mjs index a3eee74..2b16f1c 100644 --- a/test/operator-dashboard.test.mjs +++ b/test/operator-dashboard.test.mjs @@ -1684,6 +1684,102 @@ test('successful trade rows require completed outcome with linked settled invent assert.equal(funnel.counts.completed, 2); }); +test('successful trade rows include durable successful lifecycle evidence outside the recent table', () => { + const config = buildConfig(); + const bootstrap = buildDashboardBootstrap({ + config, + auth: { + authenticated: true, + subject: 'local-operator', + mode: 'stub', + roles: ['operator'], + }, + portfolioMetric: null, + inventorySnapshot: null, + marketPrice: null, + recentQuotes: [], + submissionPage: { page: 1, page_size: 20, total: 0, total_pages: 1, items: [] }, + submissionSummary: { total: 0, last_submission_at: null }, + fundingObservations: [], + recentDepositStatuses: [], + recentTradeDecisions: [], + recentExecuteTradeCommands: [], + recentExecutionResults: [], + recentQuoteOutcomes: [], + successfulQuoteLifecycle: { + found: true, + requested_identifier: 'successful_quote_lifecycle', + matched_identifier_type: 'quote_id', + matched_identifiers: { + quote_ids: ['quote-durable-success'], + decision_ids: ['decision-durable-success'], + command_ids: ['cmd-durable-success'], + }, + lookup_at: '2026-04-09T09:05:00.000Z', + recentQuotes: [], + recentTradeDecisions: [{ + observed_at: '2026-04-09T09:00:01.000Z', + payload: { + decision_id: 'decision-durable-success', + quote_id: 'quote-durable-success', + pair: config.activePair, + decision: 'actionable', + decision_reason: 'strategy_approved', + gross_edge_pct: '0.49', + eure_notional: '50', + }, + }], + recentExecuteTradeCommands: [{ + command_id: 'cmd-durable-success', + decision_id: 'decision-durable-success', + quote_id: 'quote-durable-success', + pair: config.activePair, + observed_at: '2026-04-09T09:00:02.000Z', + }], + recentExecutionResults: [{ + command_id: 'cmd-durable-success', + decision_id: 'decision-durable-success', + quote_id: 'quote-durable-success', + pair: config.activePair, + result_at: '2026-04-09T09:00:03.000Z', + status: 'submitted', + result_code: 'quote_response_ok', + }], + recentQuoteOutcomes: [{ + command_id: 'cmd-durable-success', + decision_id: 'decision-durable-success', + quote_id: 'quote-durable-success', + pair: config.activePair, + gross_edge_pct: '0.49', + eure_notional: '50', + submitted_at: '2026-04-09T09:00:03.000Z', + command_at: '2026-04-09T09:00:02.000Z', + outcome_status: 'completed', + outcome_reason: 'matched_inventory_delta', + outcome_observed_at: '2026-04-09T09:01:00.000Z', + outcome_source: 'intent_inventory_spendable_delta', + attribution_status: 'heuristic_match', + attribution_method: 'exact_asset_delta_within_window', + attributed_inventory_delta: { + observed_at: '2026-04-09T09:01:00.000Z', + delta_units: { + [config.tradingBtc.assetId]: '37014', + [config.tradingEure.assetId]: '-21000021200021200022', + }, + }, + }], + }, + recentAlertTransitions: [], + serviceSnapshots: [], + }); + + const funnel = bootstrap.strategy.strategy_state.trade_funnel; + assert.equal(funnel.successful_trade_count, 1); + assert.equal(funnel.successful_trades[0].quote_id, 'quote-durable-success'); + assert.equal(funnel.counts.completed, 1); + assert.equal(bootstrap.strategy.strategy_state.recent_lifecycle_rows[0].quote_id, 'quote-durable-success'); +}); + test('executor blocking is distinct from strategy rejection', () => { const [row] = deriveQuoteLifecycleRows({ recentTradeDecisions: [{ diff --git a/test/postgres-quote-lifecycle-statistics.test.mjs b/test/postgres-quote-lifecycle-statistics.test.mjs index d686dfd..3a5ca00 100644 --- a/test/postgres-quote-lifecycle-statistics.test.mjs +++ b/test/postgres-quote-lifecycle-statistics.test.mjs @@ -8,6 +8,7 @@ import { test('postgres quote lifecycle statistics refresh idempotently upserts state buckets by original quote window', async () => { const upserts = []; + const subjects = new Map(); let includeFailure = false; const pool = { async query(sql, params = []) { @@ -87,6 +88,23 @@ test('postgres quote lifecycle statistics refresh idempotently upserts state buc if (sql.includes('FROM quote_outcome_attributions') && sql.includes('ORDER BY COALESCE')) { return { rows: [] }; } + if (sql.includes('INSERT INTO quote_lifecycle_stat_subjects')) { + const existing = subjects.get(params[0]); + subjects.set(params[0], { + quote_id: params[0], + first_activity_at: existing?.first_activity_at || params[1], + latest_stage_at: params[2] || existing?.latest_stage_at || null, + bucket: params[3], + lifecycle_state: params[4], + reason_code: params[5], + updated_at: params[6], + payload: JSON.parse(params[7]), + }); + return { rows: [], rowCount: 1 }; + } + if (sql.includes('FROM quote_lifecycle_stat_subjects')) { + return { rows: [...subjects.values()] }; + } if (sql.includes('INSERT INTO quote_lifecycle_statistics')) { upserts.push({ sql, params }); return { rows: [], rowCount: 1 }; @@ -121,6 +139,127 @@ test('postgres quote lifecycle statistics refresh idempotently upserts state buc ); }); +test('postgres quote lifecycle statistics retain per-quote buckets after detail pruning', async () => { + const subjects = new Map(); + const allTimeUpserts = []; + let sourceAvailable = true; + const pool = { + async query(sql, params = []) { + if (sql.includes('FROM swap_demand_events') && sql.includes('ORDER BY COALESCE')) { + return { + rows: sourceAvailable ? [{ + observed_at: '2026-06-12T14:11:01.000Z', + ingested_at: '2026-06-12T14:11:01.000Z', + payload: { + quote_id: 'quote-completed', + pair: 'btc->usdc', + asset_in: 'btc', + asset_out: 'usdc', + amount_in: '1', + amount_out: '2', + }, + }] : [], + }; + } + if (sql.includes('FROM trade_decisions') && sql.includes('ORDER BY COALESCE')) { + return { + rows: sourceAvailable ? [{ + observed_at: '2026-06-12T14:11:02.000Z', + ingested_at: '2026-06-12T14:11:02.000Z', + payload: { + quote_id: 'quote-completed', + decision_id: 'decision-completed', + pair: 'btc->usdc', + decision: 'approved', + decision_reason: 'strategy_approved', + }, + }] : [], + }; + } + if (sql.includes('FROM execute_trade_commands') && sql.includes('ORDER BY COALESCE')) { + return { + rows: sourceAvailable ? [{ + observed_at: '2026-06-12T14:11:03.000Z', + ingested_at: '2026-06-12T14:11:03.000Z', + payload: { + quote_id: 'quote-completed', + decision_id: 'decision-completed', + command_id: 'cmd-completed', + pair: 'btc->usdc', + }, + }] : [], + }; + } + if (sql.includes('FROM trade_execution_results r')) { + return { + rows: sourceAvailable ? [{ + result_observed_at: '2026-06-12T14:11:10.000Z', + result_ingested_at: '2026-06-12T14:11:10.000Z', + result_payload: { + quote_id: 'quote-completed', + decision_id: 'decision-completed', + command_id: 'cmd-completed', + status: 'submitted', + result_code: 'quote_response_ok', + outcome_status: 'completed', + }, + command_payload: { + quote_id: 'quote-completed', + decision_id: 'decision-completed', + command_id: 'cmd-completed', + }, + decision_payload: { + quote_id: 'quote-completed', + decision_id: 'decision-completed', + decision: 'approved', + }, + outcome_payload: null, + }] : [], + }; + } + if (sql.includes('FROM quote_outcome_attributions') && sql.includes('ORDER BY COALESCE')) { + return { rows: [] }; + } + if (sql.includes('INSERT INTO quote_lifecycle_stat_subjects')) { + const existing = subjects.get(params[0]); + subjects.set(params[0], { + quote_id: params[0], + first_activity_at: existing?.first_activity_at || params[1], + latest_stage_at: params[2] || existing?.latest_stage_at || null, + bucket: params[3], + lifecycle_state: params[4], + reason_code: params[5], + updated_at: params[6], + payload: JSON.parse(params[7]), + }); + return { rows: [], rowCount: 1 }; + } + if (sql.includes('FROM quote_lifecycle_stat_subjects')) { + return { rows: [...subjects.values()] }; + } + if (sql.includes('INSERT INTO quote_lifecycle_statistics')) { + if (params[1] === 'all_time') allTimeUpserts.push({ params }); + return { rows: [], rowCount: 1 }; + } + throw new Error(`unexpected query: ${sql}`); + }, + }; + + await refreshQuoteLifecycleStatistics(pool, { + now: '2026-06-12T14:12:00.000Z', + }); + sourceAvailable = false; + await refreshQuoteLifecycleStatistics(pool, { + now: '2026-06-12T15:12:00.000Z', + }); + + assert.equal(allTimeUpserts.length, 2); + assert.equal(allTimeUpserts[0].params[4], 1); + assert.equal(JSON.parse(allTimeUpserts[0].params[7]).success, 1); + assert.equal(allTimeUpserts[1].params[4], 1); + assert.equal(JSON.parse(allTimeUpserts[1].params[7]).success, 1); +}); + test('postgres quote lifecycle statistics loader returns all-time and clamped grain rows', async () => { const pool = { async query(sql, params = []) { diff --git a/test/quote-lifecycle-statistics.test.mjs b/test/quote-lifecycle-statistics.test.mjs index e8ac3d3..73305da 100644 --- a/test/quote-lifecycle-statistics.test.mjs +++ b/test/quote-lifecycle-statistics.test.mjs @@ -26,6 +26,28 @@ test('quote lifecycle statistics classify states without upgrading submitted-onl }), 'success'); }); +test('quote lifecycle statistics can aggregate retained quote subject buckets', () => { + const rows = buildQuoteLifecycleStatistics({ + computedAt: '2026-06-12T14:20:00.000Z', + lifecycleRows: [{ + quote_id: 'quote-retained-success', + quote_activity_at: '2026-06-12T14:11:01.000Z', + latest_stage_at: '2026-06-12T14:11:10.000Z', + statistics_bucket: 'success', + }], + }); + + const allTime = rows.find((row) => row.grain === 'all_time'); + const fiveMinute = rows.find((row) => ( + row.grain === 'five_minute' + && row.window_start === '2026-06-12T14:10:00.000Z' + )); + assert.equal(allTime.quote_count, 1); + assert.equal(allTime.bucket_counts.success, 1); + assert.equal(fiveMinute.quote_count, 1); + assert.equal(fiveMinute.bucket_counts.success, 1); +}); + test('quote lifecycle statistics use UTC grain boundaries with Monday UTC weeks', () => { assert.deepEqual(quoteLifecycleStatisticWindow('2026-06-12T17:34:44.123Z', 'five_minute'), { grain: 'five_minute',