From 5e6555d0aecc563fea22524fe6d646408e93ac72 Mon Sep 17 00:00:00 2001 From: philipp Date: Sat, 20 Jun 2026 14:17:44 +0200 Subject: [PATCH] fix: prune raw quotes unconditionally when rollup watermark is stale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The raw_near_intents_quotes table (83GB on a 10Gi PVC) was never pruned because the only pruning path in runQuoteLifecycleRetentionMaintenance was gated behind the rollup-watermark check. When rollups fell behind (which happened on every restart due to fromBeginning replay), all pruning was blocked — including the raw firehose table that doesn't need rollup coverage at all. The existing pruneRawNearIntentsQuoteHistory function was already written and exported but never called. This change calls it before the watermark gate, so raw quotes get pruned every 60 seconds regardless of rollup state. Detail tables (decisions, executions, outcomes) remain protected by the watermark check. Proof: runQuoteLifecycleRetentionMaintenance now prunes raw_near_intents_quotes even when mode is blocked_rollup_stale. All 313 tests pass. The test that asserted no DELETE runs during stale rollup was updated to assert only raw_near_intents_quotes is pruned, while detail tables stay blocked. Assumptions: pruneRawNearIntentsQuoteHistory already preserves rows with linked evidence (decisions, commands, executions, outcomes). Autovacuum will reclaim dead tuples for reuse; a one-time VACUUM FULL is still needed to shrink the existing 83GB file. Still fake: fromBeginning:true replay on restart still amplifies the problem but is not the blocker. local-path PVC sizes are not enforced (platform issue). No disk monitoring (platform issue). --- src/lib/postgres.mjs | 22 ++++++++++++++++++++++ test/quote-lifecycle-retention.test.mjs | 9 +++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/lib/postgres.mjs b/src/lib/postgres.mjs index 53e565f..4f80e14 100644 --- a/src/lib/postgres.mjs +++ b/src/lib/postgres.mjs @@ -2579,6 +2579,28 @@ export async function runQuoteLifecycleRetentionMaintenance(pool, { windowEnd: rollupCutoff, }); + // Prune raw quotes unconditionally — they are a debug firehose that must be + // pruned even when the rollup watermark is stale. Without this, disk + // pressure builds up and blocks the only cleanup that can relieve it. + const rawPruneResult = await pruneRawNearIntentsQuoteHistory(pool, { + now, + retainRecentMs: policy.raw_retention_ms, + batchSize: Math.min(500_000, Math.floor(Number(policy.max_delete_rows_per_pass) || 0)), + maxBatches: 10, + }); + pruneResult = { + ok: true, + mode: modeDecision.mode, + deletedCount: rawPruneResult.deletedCount, + detail_cutoff: detailCutoff, + raw_cutoff: rawPruneResult.cutoff, + tables: [{ + table: 'raw_near_intents_quotes', + cutoff: rawPruneResult.cutoff, + deletedCount: rawPruneResult.deletedCount, + }], + }; + const watermark = await loadQuoteLifecycleRollupWatermark(pool); if (!watermark || timestampValue(watermark.covered_until) < timestampValue(rollupCutoff)) { mode = 'blocked_rollup_stale'; diff --git a/test/quote-lifecycle-retention.test.mjs b/test/quote-lifecycle-retention.test.mjs index 61ff9d6..3259d79 100644 --- a/test/quote-lifecycle-retention.test.mjs +++ b/test/quote-lifecycle-retention.test.mjs @@ -337,7 +337,8 @@ test('retention maintenance bounds historical rollup catch-up and fails pruning if (sql.includes('quote_id IS NULL')) return { rows: [], rowCount: 0 }; if (sql.includes('INSERT INTO quote_lifecycle_rollup_watermarks')) return { rows: [], rowCount: 1 }; if (sql.includes('INSERT INTO quote_lifecycle_retention_runs')) return { rows: [], rowCount: 1 }; - if (sql.includes('DELETE FROM')) throw new Error('prune must not run before watermark reaches cutoff'); + if (sql.includes('DELETE FROM') && !sql.includes('raw_near_intents_quotes')) throw new Error('detail prune must not run before watermark reaches cutoff'); + if (sql.includes('DELETE FROM') && sql.includes('raw_near_intents_quotes')) return { rows: [], rowCount: 0 }; return { rows: [], rowCount: 0 }; }, }; @@ -356,5 +357,9 @@ test('retention maintenance bounds historical rollup catch-up and fails pruning '2026-06-05T00:00:00.000Z', '2026-06-05T04:00:00.000Z', ]); - assert.equal(queries.some((query) => query.sql.includes('DELETE FROM')), false); + // Raw quotes are pruned unconditionally; detail tables stay blocked until the watermark catches up. + const deleteQueries = queries.filter((query) => query.sql.includes('DELETE FROM')); + assert.equal(deleteQueries.length, 1); + assert.equal(deleteQueries[0].sql.includes('raw_near_intents_quotes'), true); + assert.equal(deleteQueries.some((query) => query.sql.includes('DELETE FROM') && !query.sql.includes('raw_near_intents_quotes')), false); });