Bound raw quote retention drain

Proof: Raw NEAR Intents quote retention now keeps only a 30 minute raw firehose window and drains up to 10M stale unlinked raw rows per pass. Targeted raw retention tests, full npm test, and operator dashboard bundle build pass.

Assumptions: raw quote firehose rows are debug evidence, while normalized quote demand, decisions, commands, executor results, outcomes, inventory, pricing, and DB config remain the durable trading evidence.

Still fake: venue-native terminal fill ids and fee-complete realized PnL remain unavailable; raw quote firehose rows truncated during emergency recovery are intentionally no longer readable.

src/apps/history-writer.mjs

@@ -121,9 +121,10 @@ const topics = [
 const rawQuoteTopics = [
   config.kafkaTopicRawNearIntentsQuote,
 ];
-const rawQuoteHistoryPruneIntervalMs = 15 * 60 * 1000;
+const rawQuoteHistoryPruneIntervalMs = 60 * 1000;
-const rawQuoteHistoryRetainRecentMs = 6 * 60 * 60 * 1000;
+const rawQuoteHistoryRetainRecentMs = 30 * 60 * 1000;
-const rawQuoteHistoryPruneBatchSize = 100_000;
+const rawQuoteHistoryPruneBatchSize = 500_000;
+const rawQuoteHistoryPruneMaxBatches = 20;
 const liveEvidenceTopics = [
   config.kafkaTopicNormSwapDemand,
   config.kafkaTopicDecisionTradeDecision,
@@ -312,6 +313,7 @@ async function maybePruneRawQuoteHistory({ force = false } = {}) {
       now: new Date(nowMs).toISOString(),
       retainRecentMs: rawQuoteHistoryRetainRecentMs,
       batchSize: rawQuoteHistoryPruneBatchSize,
+      maxBatches: rawQuoteHistoryPruneMaxBatches,
     });
     state.last_raw_quote_prune_at = new Date(nowMs).toISOString();
     state.raw_quote_prune_deleted_count += result.deletedCount;

src/lib/postgres.mjs

@@ -2127,8 +2127,9 @@ export async function insertHistoryEvents(pool, entries = []) {
 
 export async function pruneRawNearIntentsQuoteHistory(pool, {
   now = new Date().toISOString(),
-  retainRecentMs = 6 * 60 * 60 * 1000,
+  retainRecentMs = 30 * 60 * 1000,
   batchSize = 100_000,
+  maxBatches = 1,
 } = {}) {
   const nowMs = Date.parse(now);
   if (!Number.isFinite(nowMs)) throw new Error('now must be a valid timestamp');
@@ -2136,8 +2137,12 @@ export async function pruneRawNearIntentsQuoteHistory(pool, {
     throw new Error('retain_recent_ms must be a positive integer');
   }
   const boundedBatchSize = Math.max(1, Math.min(500_000, Math.floor(Number(batchSize) || 0)));
+  const boundedMaxBatches = Math.max(1, Math.min(100, Math.floor(Number(maxBatches) || 0)));
   const cutoff = new Date(nowMs - retainRecentMs).toISOString();
 
+  let deletedCount = 0;
+  let batches = 0;
+  for (let batch = 0; batch < boundedMaxBatches; batch += 1) {
     const result = await pool.query(
       `
         WITH stale_raw_quotes AS (
@@ -2169,12 +2174,19 @@ export async function pruneRawNearIntentsQuoteHistory(pool, {
       `,
       [cutoff, boundedBatchSize],
     );
+    const batchDeleted = Number(result.rowCount || 0);
+    deletedCount += batchDeleted;
+    batches += 1;
+    if (batchDeleted < boundedBatchSize) break;
+  }
 
   return {
-    deletedCount: Number(result.rowCount || 0),
+    deletedCount,
     cutoff,
     retainRecentMs,
     batchSize: boundedBatchSize,
+    maxBatches: boundedMaxBatches,
+    batches,
   };
 }
 

test/history-writer-static.test.mjs

@@ -20,8 +20,12 @@ test('history writer replays durable topics but joins the raw quote firehose liv
   assert.match(source, /runHistoryConsumer\(rawQuoteConsumer\)/);
   assert.match(source, /eachBatch/);
   assert.match(source, /insertHistoryEvents/);
-  assert.match(source, /rawQuoteHistoryRetainRecentMs\s*=\s*6 \* 60 \* 60 \* 1000/);
+  assert.match(source, /rawQuoteHistoryPruneIntervalMs\s*=\s*60 \* 1000/);
+  assert.match(source, /rawQuoteHistoryRetainRecentMs\s*=\s*30 \* 60 \* 1000/);
+  assert.match(source, /rawQuoteHistoryPruneBatchSize\s*=\s*500_000/);
+  assert.match(source, /rawQuoteHistoryPruneMaxBatches\s*=\s*20/);
   assert.match(source, /pruneRawNearIntentsQuoteHistory/);
+  assert.match(source, /maxBatches:\s*rawQuoteHistoryPruneMaxBatches/);
   assert.match(source, /batch\.topic === config\.kafkaTopicRawNearIntentsQuote/);
 });
 

test/inventory-and-history.test.mjs

@@ -128,6 +128,35 @@ test('raw quote retention preserves rows linked to maker lifecycle evidence', as
   assert.deepEqual(queries[0].params, ['2026-05-21T06:00:00.000Z', 123]);
 });
 
+test('raw quote retention drains multiple bounded batches when firehose backlog exceeds one batch', async () => {
+  const rowCounts = [500_000, 500_000, 12];
+  const queries = [];
+  const pool = {
+    async query(sql, params) {
+      queries.push({ sql, params });
+      return { rows: [], rowCount: rowCounts.shift() ?? 0 };
+    },
+  };
+
+  const result = await pruneRawNearIntentsQuoteHistory(pool, {
+    now: '2026-05-21T12:00:00.000Z',
+    retainRecentMs: 30 * 60 * 1000,
+    batchSize: 500_000,
+    maxBatches: 20,
+  });
+
+  assert.equal(result.deletedCount, 1_000_012);
+  assert.equal(result.cutoff, '2026-05-21T11:30:00.000Z');
+  assert.equal(result.batchSize, 500_000);
+  assert.equal(result.maxBatches, 20);
+  assert.equal(result.batches, 3);
+  assert.equal(queries.length, 3);
+  for (const query of queries) {
+    assert.match(query.sql, /DELETE FROM raw_near_intents_quotes/);
+    assert.deepEqual(query.params, ['2026-05-21T11:30:00.000Z', 500_000]);
+  }
+});
+
 function historyEvent(eventId, payload) {
   return {
     event_id: eventId,