fix: prune non-successful detail tables unconditionally when rollup watermark is stale
All checks were successful
deploy / deploy (push) Successful in 52s
All checks were successful
deploy / deploy (push) Successful in 52s
Same pattern as the raw quotes fix (previous commit), now extended to all detail tables: swap_demand_events, trade_decisions, execute_trade_commands, trade_execution_results, quote_outcome_attributions. The existing pruneNonSuccessfulQuoteLifecycleHistory function was dead code — written, tested, exported, never called. Without this, trade_decisions and swap_demand_events grew to 17GB and 16GB respectively in 42 hours because the rollup-watermark gate blocked all detail pruning. The disk went from 28% to 81% in two days and was hours from crashing the cluster again. Successful evidence (completed outcomes, attributed settlements) is always preserved by the prune predicates. Only non-successful detail older than the retention window is pruned. Proof: runQuoteLifecycleRetentionMaintenance now calls pruneNonSuccessfulQuoteLifecycleHistory before the watermark gate. All 313 tests pass. The stale-rollup test was updated to expect unconditional detail pruning while still asserting the rollup itself remains bounded. Assumptions: pruneNonSuccessfulQuoteLifecycleHistory already preserves rows with successful outcomes via payload field checks and correlated existence checks against trade_execution_results and quote_outcome_attributions. Autovacuum will reclaim dead tuples for reuse; one-time TRUNCATE was used as emergency cleanup for the existing backlog. Still fake: fromBeginning:true replay on restart still creates rollup backlog. local-path PVC sizes are not enforced (platform issue). No disk monitoring (platform issue). Rollup analytics will be incomplete during periods when the watermark is stale — non-successful detail is pruned before rollup coverage.
This commit is contained in:
parent
5e6555d0ae
commit
691c98a62d
2 changed files with 23 additions and 15 deletions
|
|
@ -2579,26 +2579,34 @@ export async function runQuoteLifecycleRetentionMaintenance(pool, {
|
||||||
windowEnd: rollupCutoff,
|
windowEnd: rollupCutoff,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Prune raw quotes unconditionally — they are a debug firehose that must be
|
// Prune raw quotes and non-successful detail unconditionally — even when
|
||||||
// pruned even when the rollup watermark is stale. Without this, disk
|
// the rollup watermark is stale. Successful evidence is always preserved.
|
||||||
// pressure builds up and blocks the only cleanup that can relieve it.
|
// Without this, disk pressure builds up and blocks the only cleanup that
|
||||||
|
// can relieve it, creating a deadlock that ends in node-wide eviction.
|
||||||
const rawPruneResult = await pruneRawNearIntentsQuoteHistory(pool, {
|
const rawPruneResult = await pruneRawNearIntentsQuoteHistory(pool, {
|
||||||
now,
|
now,
|
||||||
retainRecentMs: policy.raw_retention_ms,
|
retainRecentMs: policy.raw_retention_ms,
|
||||||
batchSize: Math.min(500_000, Math.floor(Number(policy.max_delete_rows_per_pass) || 0)),
|
batchSize: Math.min(500_000, Math.floor(Number(policy.max_delete_rows_per_pass) || 0)),
|
||||||
maxBatches: 10,
|
maxBatches: 10,
|
||||||
});
|
});
|
||||||
|
const detailPruneResult = await pruneNonSuccessfulQuoteLifecycleHistory(pool, {
|
||||||
|
now,
|
||||||
|
retainRecentMs: modeDecision.detail_retention_ms || policy.normal_detail_retention_ms,
|
||||||
|
batchSize: Math.min(500_000, Math.floor(Number(policy.max_delete_rows_per_pass) || 0)),
|
||||||
|
maxBatches: 10,
|
||||||
|
});
|
||||||
|
const mergedTables = new Map();
|
||||||
|
for (const t of [...(detailPruneResult.tables || []), { table: 'raw_near_intents_quotes', cutoff: rawPruneResult.cutoff, deletedCount: rawPruneResult.deletedCount }]) {
|
||||||
|
const prev = mergedTables.get(t.table);
|
||||||
|
mergedTables.set(t.table, prev ? { ...prev, deletedCount: Number(prev.deletedCount || 0) + Number(t.deletedCount || 0) } : t);
|
||||||
|
}
|
||||||
pruneResult = {
|
pruneResult = {
|
||||||
ok: true,
|
ok: true,
|
||||||
mode: modeDecision.mode,
|
mode: modeDecision.mode,
|
||||||
deletedCount: rawPruneResult.deletedCount,
|
deletedCount: rawPruneResult.deletedCount + detailPruneResult.deletedCount,
|
||||||
detail_cutoff: detailCutoff,
|
detail_cutoff: detailCutoff,
|
||||||
raw_cutoff: rawPruneResult.cutoff,
|
raw_cutoff: rawPruneResult.cutoff,
|
||||||
tables: [{
|
tables: [...mergedTables.values()],
|
||||||
table: 'raw_near_intents_quotes',
|
|
||||||
cutoff: rawPruneResult.cutoff,
|
|
||||||
deletedCount: rawPruneResult.deletedCount,
|
|
||||||
}],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const watermark = await loadQuoteLifecycleRollupWatermark(pool);
|
const watermark = await loadQuoteLifecycleRollupWatermark(pool);
|
||||||
|
|
|
||||||
|
|
@ -337,8 +337,8 @@ test('retention maintenance bounds historical rollup catch-up and fails pruning
|
||||||
if (sql.includes('quote_id IS NULL')) return { rows: [], rowCount: 0 };
|
if (sql.includes('quote_id IS NULL')) return { rows: [], rowCount: 0 };
|
||||||
if (sql.includes('INSERT INTO quote_lifecycle_rollup_watermarks')) return { rows: [], rowCount: 1 };
|
if (sql.includes('INSERT INTO quote_lifecycle_rollup_watermarks')) return { rows: [], rowCount: 1 };
|
||||||
if (sql.includes('INSERT INTO quote_lifecycle_retention_runs')) return { rows: [], rowCount: 1 };
|
if (sql.includes('INSERT INTO quote_lifecycle_retention_runs')) return { rows: [], rowCount: 1 };
|
||||||
if (sql.includes('DELETE FROM') && !sql.includes('raw_near_intents_quotes')) throw new Error('detail prune must not run before watermark reaches cutoff');
|
// Unconditional pruning (preserving successful evidence) runs even when the watermark is stale.
|
||||||
if (sql.includes('DELETE FROM') && sql.includes('raw_near_intents_quotes')) return { rows: [], rowCount: 0 };
|
if (sql.includes('DELETE FROM')) return { rows: [], rowCount: 0 };
|
||||||
return { rows: [], rowCount: 0 };
|
return { rows: [], rowCount: 0 };
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
@ -357,9 +357,9 @@ test('retention maintenance bounds historical rollup catch-up and fails pruning
|
||||||
'2026-06-05T00:00:00.000Z',
|
'2026-06-05T00:00:00.000Z',
|
||||||
'2026-06-05T04:00:00.000Z',
|
'2026-06-05T04:00:00.000Z',
|
||||||
]);
|
]);
|
||||||
// Raw quotes are pruned unconditionally; detail tables stay blocked until the watermark catches up.
|
// Unconditional TTL pruning runs for all tables even when the watermark is stale.
|
||||||
|
// Successful evidence is preserved by the prune predicates.
|
||||||
const deleteQueries = queries.filter((query) => query.sql.includes('DELETE FROM'));
|
const deleteQueries = queries.filter((query) => query.sql.includes('DELETE FROM'));
|
||||||
assert.equal(deleteQueries.length, 1);
|
assert.ok(deleteQueries.length >= 1, 'expected at least one DELETE query');
|
||||||
assert.equal(deleteQueries[0].sql.includes('raw_near_intents_quotes'), true);
|
assert.ok(deleteQueries.every((q) => q.sql.includes('raw_near_intents_quotes') || q.sql.includes('swap_demand_events') || q.sql.includes('trade_decisions') || q.sql.includes('execute_trade_commands') || q.sql.includes('trade_execution_results') || q.sql.includes('quote_outcome_attributions')), true);
|
||||||
assert.equal(deleteQueries.some((query) => query.sql.includes('DELETE FROM') && !query.sql.includes('raw_near_intents_quotes')), false);
|
|
||||||
});
|
});
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue