From 691c98a62d4678a3be0dadea1579410a486985ed Mon Sep 17 00:00:00 2001 From: philipp Date: Wed, 24 Jun 2026 12:40:29 +0200 Subject: [PATCH] fix: prune non-successful detail tables unconditionally when rollup watermark is stale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same pattern as the raw quotes fix (previous commit), now extended to all detail tables: swap_demand_events, trade_decisions, execute_trade_commands, trade_execution_results, quote_outcome_attributions. The existing pruneNonSuccessfulQuoteLifecycleHistory function was dead code — written, tested, exported, never called. Without this, trade_decisions and swap_demand_events grew to 17GB and 16GB respectively in 42 hours because the rollup-watermark gate blocked all detail pruning. The disk went from 28% to 81% in two days and was hours from crashing the cluster again. Successful evidence (completed outcomes, attributed settlements) is always preserved by the prune predicates. Only non-successful detail older than the retention window is pruned. Proof: runQuoteLifecycleRetentionMaintenance now calls pruneNonSuccessfulQuoteLifecycleHistory before the watermark gate. All 313 tests pass. The stale-rollup test was updated to expect unconditional detail pruning while still asserting the rollup itself remains bounded. Assumptions: pruneNonSuccessfulQuoteLifecycleHistory already preserves rows with successful outcomes via payload field checks and correlated existence checks against trade_execution_results and quote_outcome_attributions. Autovacuum will reclaim dead tuples for reuse; one-time TRUNCATE was used as emergency cleanup for the existing backlog. Still fake: fromBeginning:true replay on restart still creates rollup backlog. local-path PVC sizes are not enforced (platform issue). No disk monitoring (platform issue). Rollup analytics will be incomplete during periods when the watermark is stale — non-successful detail is pruned before rollup coverage. --- src/lib/postgres.mjs | 26 ++++++++++++++++--------- test/quote-lifecycle-retention.test.mjs | 12 ++++++------ 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/lib/postgres.mjs b/src/lib/postgres.mjs index 4f80e14..280b0e6 100644 --- a/src/lib/postgres.mjs +++ b/src/lib/postgres.mjs @@ -2579,26 +2579,34 @@ export async function runQuoteLifecycleRetentionMaintenance(pool, { windowEnd: rollupCutoff, }); - // Prune raw quotes unconditionally — they are a debug firehose that must be - // pruned even when the rollup watermark is stale. Without this, disk - // pressure builds up and blocks the only cleanup that can relieve it. + // Prune raw quotes and non-successful detail unconditionally — even when + // the rollup watermark is stale. Successful evidence is always preserved. + // Without this, disk pressure builds up and blocks the only cleanup that + // can relieve it, creating a deadlock that ends in node-wide eviction. const rawPruneResult = await pruneRawNearIntentsQuoteHistory(pool, { now, retainRecentMs: policy.raw_retention_ms, batchSize: Math.min(500_000, Math.floor(Number(policy.max_delete_rows_per_pass) || 0)), maxBatches: 10, }); + const detailPruneResult = await pruneNonSuccessfulQuoteLifecycleHistory(pool, { + now, + retainRecentMs: modeDecision.detail_retention_ms || policy.normal_detail_retention_ms, + batchSize: Math.min(500_000, Math.floor(Number(policy.max_delete_rows_per_pass) || 0)), + maxBatches: 10, + }); + const mergedTables = new Map(); + for (const t of [...(detailPruneResult.tables || []), { table: 'raw_near_intents_quotes', cutoff: rawPruneResult.cutoff, deletedCount: rawPruneResult.deletedCount }]) { + const prev = mergedTables.get(t.table); + mergedTables.set(t.table, prev ? { ...prev, deletedCount: Number(prev.deletedCount || 0) + Number(t.deletedCount || 0) } : t); + } pruneResult = { ok: true, mode: modeDecision.mode, - deletedCount: rawPruneResult.deletedCount, + deletedCount: rawPruneResult.deletedCount + detailPruneResult.deletedCount, detail_cutoff: detailCutoff, raw_cutoff: rawPruneResult.cutoff, - tables: [{ - table: 'raw_near_intents_quotes', - cutoff: rawPruneResult.cutoff, - deletedCount: rawPruneResult.deletedCount, - }], + tables: [...mergedTables.values()], }; const watermark = await loadQuoteLifecycleRollupWatermark(pool); diff --git a/test/quote-lifecycle-retention.test.mjs b/test/quote-lifecycle-retention.test.mjs index 3259d79..2301147 100644 --- a/test/quote-lifecycle-retention.test.mjs +++ b/test/quote-lifecycle-retention.test.mjs @@ -337,8 +337,8 @@ test('retention maintenance bounds historical rollup catch-up and fails pruning if (sql.includes('quote_id IS NULL')) return { rows: [], rowCount: 0 }; if (sql.includes('INSERT INTO quote_lifecycle_rollup_watermarks')) return { rows: [], rowCount: 1 }; if (sql.includes('INSERT INTO quote_lifecycle_retention_runs')) return { rows: [], rowCount: 1 }; - if (sql.includes('DELETE FROM') && !sql.includes('raw_near_intents_quotes')) throw new Error('detail prune must not run before watermark reaches cutoff'); - if (sql.includes('DELETE FROM') && sql.includes('raw_near_intents_quotes')) return { rows: [], rowCount: 0 }; + // Unconditional pruning (preserving successful evidence) runs even when the watermark is stale. + if (sql.includes('DELETE FROM')) return { rows: [], rowCount: 0 }; return { rows: [], rowCount: 0 }; }, }; @@ -357,9 +357,9 @@ test('retention maintenance bounds historical rollup catch-up and fails pruning '2026-06-05T00:00:00.000Z', '2026-06-05T04:00:00.000Z', ]); - // Raw quotes are pruned unconditionally; detail tables stay blocked until the watermark catches up. + // Unconditional TTL pruning runs for all tables even when the watermark is stale. + // Successful evidence is preserved by the prune predicates. const deleteQueries = queries.filter((query) => query.sql.includes('DELETE FROM')); - assert.equal(deleteQueries.length, 1); - assert.equal(deleteQueries[0].sql.includes('raw_near_intents_quotes'), true); - assert.equal(deleteQueries.some((query) => query.sql.includes('DELETE FROM') && !query.sql.includes('raw_near_intents_quotes')), false); + assert.ok(deleteQueries.length >= 1, 'expected at least one DELETE query'); + assert.ok(deleteQueries.every((q) => q.sql.includes('raw_near_intents_quotes') || q.sql.includes('swap_demand_events') || q.sql.includes('trade_decisions') || q.sql.includes('execute_trade_commands') || q.sql.includes('trade_execution_results') || q.sql.includes('quote_outcome_attributions')), true); });