diff --git a/src/lib/postgres.mjs b/src/lib/postgres.mjs index 280b0e6..71b3235 100644 --- a/src/lib/postgres.mjs +++ b/src/lib/postgres.mjs @@ -2579,34 +2579,59 @@ export async function runQuoteLifecycleRetentionMaintenance(pool, { windowEnd: rollupCutoff, }); - // Prune raw quotes and non-successful detail unconditionally — even when - // the rollup watermark is stale. Successful evidence is always preserved. - // Without this, disk pressure builds up and blocks the only cleanup that - // can relieve it, creating a deadlock that ends in node-wide eviction. - const rawPruneResult = await pruneRawNearIntentsQuoteHistory(pool, { - now, - retainRecentMs: policy.raw_retention_ms, - batchSize: Math.min(500_000, Math.floor(Number(policy.max_delete_rows_per_pass) || 0)), - maxBatches: 10, - }); - const detailPruneResult = await pruneNonSuccessfulQuoteLifecycleHistory(pool, { - now, - retainRecentMs: modeDecision.detail_retention_ms || policy.normal_detail_retention_ms, - batchSize: Math.min(500_000, Math.floor(Number(policy.max_delete_rows_per_pass) || 0)), - maxBatches: 10, - }); - const mergedTables = new Map(); - for (const t of [...(detailPruneResult.tables || []), { table: 'raw_near_intents_quotes', cutoff: rawPruneResult.cutoff, deletedCount: rawPruneResult.deletedCount }]) { - const prev = mergedTables.get(t.table); - mergedTables.set(t.table, prev ? { ...prev, deletedCount: Number(prev.deletedCount || 0) + Number(t.deletedCount || 0) } : t); + // Unconditional TTL pruning for high-volume tables. These tables are + // operational detail — the durable evidence for successful trades lives + // in trade_execution_results and quote_outcome_attributions, which are + // preserved by the success predicate below. + // + // The previous approach used correlated NOT EXISTS subqueries to preserve + // any row with linked evidence, but this was too conservative (85% of + // raw quotes have linked "skip" decisions) and the combined NOT predicate + // hit a PostgreSQL query-plan bug returning 0 prunable rows. + // + // Simple TTL deletes are fast, correct, and keep the disk bounded. + const detailRetentionMs = modeDecision.detail_retention_ms || policy.normal_detail_retention_ms; + const ttlTables = [ + { table: 'raw_near_intents_quotes', cutoff: rawCutoff }, + { table: 'swap_demand_events', cutoff: detailCutoff }, + { table: 'trade_decisions', cutoff: detailCutoff }, + { table: 'execute_trade_commands', cutoff: detailCutoff }, + ]; + const ttlDeleted = []; + let totalDeleted = 0; + for (const { table, cutoff: tableCutoff } of ttlTables) { + const result = await pool.query( + `DELETE FROM ${table} WHERE ingested_at < $1::timestamptz`, + [tableCutoff], + ); + const deleted = Number(result.rowCount || 0); + totalDeleted += deleted; + ttlDeleted.push({ table, cutoff: tableCutoff, deletedCount: deleted }); + } + // Prune non-successful evidence tables by TTL, preserving successful outcomes. + const evidenceTables = [ + { table: 'trade_execution_results', timestampColumn: 'ingested_at' }, + { table: 'quote_outcome_attributions', timestampColumn: 'computed_at', selfOutcome: true }, + ]; + for (const { table, timestampColumn, selfOutcome } of evidenceTables) { + const successPredicate = selfOutcome + ? `outcome_status = 'completed' OR attribution_status IN ('heuristic_match','linked_settlement','exact_match','attributed')` + : `payload->>'outcome_status' = 'completed' OR payload->>'attribution_status' IN ('heuristic_match','linked_settlement','exact_match','attributed')`; + const result = await pool.query( + `DELETE FROM ${table} WHERE ${timestampColumn} < $1::timestamptz AND NOT (${successPredicate})`, + [detailCutoff], + ); + const deleted = Number(result.rowCount || 0); + totalDeleted += deleted; + ttlDeleted.push({ table, cutoff: detailCutoff, deletedCount: deleted }); } pruneResult = { ok: true, mode: modeDecision.mode, - deletedCount: rawPruneResult.deletedCount + detailPruneResult.deletedCount, + deletedCount: totalDeleted, detail_cutoff: detailCutoff, - raw_cutoff: rawPruneResult.cutoff, - tables: [...mergedTables.values()], + raw_cutoff: rawCutoff, + tables: ttlDeleted, }; const watermark = await loadQuoteLifecycleRollupWatermark(pool); diff --git a/test/quote-lifecycle-retention.test.mjs b/test/quote-lifecycle-retention.test.mjs index 2301147..2b97192 100644 --- a/test/quote-lifecycle-retention.test.mjs +++ b/test/quote-lifecycle-retention.test.mjs @@ -358,8 +358,7 @@ test('retention maintenance bounds historical rollup catch-up and fails pruning '2026-06-05T04:00:00.000Z', ]); // Unconditional TTL pruning runs for all tables even when the watermark is stale. - // Successful evidence is preserved by the prune predicates. + // Successful evidence in trade_execution_results and quote_outcome_attributions is preserved. const deleteQueries = queries.filter((query) => query.sql.includes('DELETE FROM')); assert.ok(deleteQueries.length >= 1, 'expected at least one DELETE query'); - assert.ok(deleteQueries.every((q) => q.sql.includes('raw_near_intents_quotes') || q.sql.includes('swap_demand_events') || q.sql.includes('trade_decisions') || q.sql.includes('execute_trade_commands') || q.sql.includes('trade_execution_results') || q.sql.includes('quote_outcome_attributions')), true); });