philipp/unrip
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Split raw history consumer
All checks were successful
deploy / deploy (push) Successful in 45s
Details

Browse source 
Proof: raw quote persistence now uses a dedicated history consumer group so raw quote firehose volume cannot starve durable normalized quote, decision, command, result, and outcome evidence topics in the main history-writer group.

Assumptions: raw quote persistence can join live in a dedicated group without changing event schemas or strategy behavior; no live pair, edge, notional, inventory, arming, or response-policy settings are changed.

Still fake: venue-native terminal fill ids and fee-complete realized PnL remain unavailable; historical backlog catch-up still depends on Kafka and Postgres throughput after deployment.
This commit is contained in:
philipp 2026-05-19 16:00:54 +02:00
parent 348c4f9b0b
commit 5f2380fdc0
2 changed files with 101 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

156
src/apps/history-writer.mjs
View file

@ -84,9 +84,14 @@ const consumer = await createConsumer({
clientId: config.kafkaClientId, clientId: config.kafkaClientId,
logger, logger,
}); });
const rawQuoteConsumer = await createConsumer({
groupId: `${config.kafkaConsumerGroupHistory}-raw`,
brokers: config.kafkaBrokers,
clientId: config.kafkaClientId,
logger,
});
const topics = [ const topics = [
config.kafkaTopicRawNearIntentsQuote,
config.kafkaTopicNormSwapDemand, config.kafkaTopicNormSwapDemand,
config.kafkaTopicRefMarketPrice, config.kafkaTopicRefMarketPrice,
config.kafkaTopicStateIntentInventory, config.kafkaTopicStateIntentInventory,
@ -98,6 +103,9 @@ const topics = [
config.kafkaTopicCmdExecuteTrade, config.kafkaTopicCmdExecuteTrade,
config.kafkaTopicExecTradeResult, config.kafkaTopicExecTradeResult,
]; ];
const rawQuoteTopics = [
config.kafkaTopicRawNearIntentsQuote,
];
const portfolioMetricTopics = new Set([ const portfolioMetricTopics = new Set([
config.kafkaTopicRefMarketPrice, config.kafkaTopicRefMarketPrice,
config.kafkaTopicStateIntentInventory, config.kafkaTopicStateIntentInventory,
@ -115,11 +123,17 @@ const intentRequestOutcomeTopics = new Set([
]); ]);
for (const topic of topics) { for (const topic of topics) {
// Raw quote volume is a live firehose; replaying retained history can starve
// durable strategy/execution topics and exhaust the writer.
await consumer.subscribe({ await consumer.subscribe({
topic, topic,
fromBeginning: topic !== config.kafkaTopicRawNearIntentsQuote, fromBeginning: true,
});
}
for (const topic of rawQuoteTopics) {
// Raw quote volume is a live firehose. It gets a dedicated consumer group so
// raw storage cannot starve durable strategy/execution topics.
await rawQuoteConsumer.subscribe({
topic,
fromBeginning: false,
}); });
} }
@ -164,70 +178,75 @@ await refreshIntentRequestOutcomeAttributions().catch((error) => {
state.intent_request_outcomes_error = serializeError(error); state.intent_request_outcomes_error = serializeError(error);
}); });
await consumer.run({ await runHistoryConsumer(consumer);
eachBatch: async ({ batch, heartbeat }) => { await runHistoryConsumer(rawQuoteConsumer);
if (state.paused) return;
const contexts = []; async function runHistoryConsumer(historyConsumer) {
const batchEntries = []; await historyConsumer.run({
eachBatch: async ({ batch, heartbeat }) => {
if (state.paused) return;
for (const message of batch.messages) { const contexts = [];
if (!message.value) continue; const batchEntries = [];
try {
const event = parseEventMessage(message.value.toString());
const routed = routeHistoryRecord({ topic: batch.topic, event });
const context = {
topic: batch.topic,
partition: batch.partition,
message,
event,
routed,
writeResult: null,
};
contexts.push(context); for (const message of batch.messages) {
if (batch.topic === config.kafkaTopicOpsEnvironmentStatus) { if (!message.value) continue;
context.writeResult = await insertEnvironmentStatusChange(pool, { try {
const event = parseEventMessage(message.value.toString());
const routed = routeHistoryRecord({ topic: batch.topic, event });
const context = {
topic: batch.topic, topic: batch.topic,
partition: batch.partition,
message,
event, event,
record: routed.record, routed,
}); writeResult: null,
} else { };
batchEntries.push({
table: routed.table, contexts.push(context);
topic: batch.topic, if (batch.topic === config.kafkaTopicOpsEnvironmentStatus) {
event, context.writeResult = await insertEnvironmentStatusChange(pool, {
record: routed.record, topic: batch.topic,
}); event,
record: routed.record,
});
} else {
batchEntries.push({
table: routed.table,
topic: batch.topic,
event,
record: routed.record,
});
}
} catch (error) {
recordHistoryError(batch.topic, error);
} }
}
let insertedEventIds = new Set();
try {
({ insertedEventIds } = await insertHistoryEvents(pool, batchEntries));
} catch (error) { } catch (error) {
recordHistoryError(batch.topic, error); recordHistoryError(batch.topic, error);
throw error;
} }
}
let insertedEventIds = new Set(); for (const context of contexts) {
try { if (!context.writeResult) {
({ insertedEventIds } = await insertHistoryEvents(pool, batchEntries)); context.writeResult = {
} catch (error) { inserted: insertedEventIds.has(context.event.event_id),
recordHistoryError(batch.topic, error); };
throw error; }
} await handleWrittenHistoryEvent(context);
await heartbeat();
for (const context of contexts) {
if (!context.writeResult) {
context.writeResult = {
inserted: insertedEventIds.has(context.event.event_id),
};
} }
await handleWrittenHistoryEvent(context);
await heartbeat();
}
if (state.draining) { if (state.draining) {
setTimeout(() => shutdown(), 0); setTimeout(() => shutdown(), 0);
} }
}, },
}); });
}
async function handleWrittenHistoryEvent({ async function handleWrittenHistoryEvent({
topic, topic,
@ -399,7 +418,7 @@ const controlApi = startControlApi({
path: '/pause', path: '/pause',
handler: () => { handler: () => {
state.paused = true; state.paused = true;
consumer.pause(topics.map((topic) => ({ topic }))); pauseConsumers();
return { ok: true, paused: true }; return { ok: true, paused: true };
}, },
}, },
@ -408,7 +427,7 @@ const controlApi = startControlApi({
path: '/resume', path: '/resume',
handler: () => { handler: () => {
state.paused = false; state.paused = false;
consumer.resume(topics.map((topic) => ({ topic }))); resumeConsumers();
return { ok: true, paused: false }; return { ok: true, paused: false };
}, },
}, },
@ -418,7 +437,7 @@ const controlApi = startControlApi({
handler: () => { handler: () => {
state.draining = true; state.draining = true;
state.paused = true; state.paused = true;
consumer.pause(topics.map((topic) => ({ topic }))); pauseConsumers();
setTimeout(() => shutdown(), 0); setTimeout(() => shutdown(), 0);
return { ok: true, draining: true }; return { ok: true, draining: true };
}, },
@ -568,9 +587,26 @@ function summarizePortfolioMetric(metric) {
}; };
} }
function topicRefs(topicNames) {
return topicNames.map((topic) => ({ topic }));
}
function pauseConsumers() {
consumer.pause(topicRefs(topics));
rawQuoteConsumer.pause(topicRefs(rawQuoteTopics));
}
function resumeConsumers() {
consumer.resume(topicRefs(topics));
rawQuoteConsumer.resume(topicRefs(rawQuoteTopics));
}
async function shutdown() { async function shutdown() {
await controlApi.close().catch(() => {}); await controlApi.close().catch(() => {});
await consumer.disconnect(); await Promise.allSettled([
consumer.disconnect(),
rawQuoteConsumer.disconnect(),
]);
await pool.end(); await pool.end();
process.exit(0); process.exit(0);
} }

6
test/history-writer-static.test.mjs
View file

@ -5,8 +5,12 @@ import { readFileSync } from 'node:fs';
const source = readFileSync(new URL('../src/apps/history-writer.mjs', import.meta.url), 'utf8'); const source = readFileSync(new URL('../src/apps/history-writer.mjs', import.meta.url), 'utf8');
test('history writer replays durable topics but joins the raw quote firehose live', () => { test('history writer replays durable topics but joins the raw quote firehose live', () => {
assert.match(source, /fromBeginning:\s*topic !== config\.kafkaTopicRawNearIntentsQuote/); assert.match(source, /groupId:\s*`\$\{config\.kafkaConsumerGroupHistory\}-raw`/);
assert.match(source, /rawQuoteConsumer\.subscribe\(\{[\s\S]+fromBeginning:\s*false/);
assert.match(source, /consumer\.subscribe\(\{[\s\S]+fromBeginning:\s*true/);
assert.match(source, /Raw quote volume is a live firehose/); assert.match(source, /Raw quote volume is a live firehose/);
assert.match(source, /runHistoryConsumer\(consumer\)/);
assert.match(source, /runHistoryConsumer\(rawQuoteConsumer\)/);
assert.match(source, /eachBatch/); assert.match(source, /eachBatch/);
assert.match(source, /insertHistoryEvents/); assert.match(source, /insertHistoryEvents/);
}); });