diff --git a/check-matches-small.mjs b/check-matches-small.mjs deleted file mode 100644 index 3073685..0000000 --- a/check-matches-small.mjs +++ /dev/null @@ -1,54 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { deriveQuoteOutcomeRecords } from './src/core/quote-outcomes.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - - // Wait a bit for connection - await new Promise(r => setTimeout(r, 1000)); - - console.log("Fetching submissions..."); - const expectedDeltasRows = await pool.query(` - SELECT - quote_id, - COALESCE(observed_at, ingested_at) as submitted_at, - payload->>'expected_inventory_delta' as expected_inventory_delta - FROM trade_execution_results - WHERE payload->>'status' = 'submitted' - ORDER BY COALESCE(observed_at, ingested_at) DESC - LIMIT 2000 - `); - const submissions = expectedDeltasRows.rows.map(r => ({ - quote_id: r.quote_id, - submitted_at: r.submitted_at, - expected_inventory_delta: typeof r.expected_inventory_delta === 'string' && r.expected_inventory_delta ? JSON.parse(r.expected_inventory_delta) : r.expected_inventory_delta - })); - - console.log("Fetching inventory...", submissions.length); - const inventoryRows = await pool.query(` - SELECT - event_id as inventory_id, - observed_at, - payload - FROM intent_inventory_snapshots - ORDER BY COALESCE(observed_at, ingested_at) DESC - LIMIT 5000 - `); - const inventorySnapshots = inventoryRows.rows.map(r => ({ - inventory_id: r.inventory_id, - observed_at: r.observed_at, - asset_balances: r.payload?.asset_balances || {}, - })); - - console.log("Checking for matches...", submissions.length, inventorySnapshots.length); - const records = deriveQuoteOutcomeRecords({ - submissions, - inventorySnapshots, - }); - - const completed = records.filter(r => r.outcome_status === 'completed'); - console.log("Found", completed.length, "completed trades out of", records.length); - process.exit(0); -} - -main().catch(console.error); diff --git a/check-matches.mjs b/check-matches.mjs deleted file mode 100644 index aeaea98..0000000 --- a/check-matches.mjs +++ /dev/null @@ -1,49 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { deriveQuoteOutcomeRecords } from './src/core/quote-outcomes.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - - const expectedDeltasRows = await pool.query(` - SELECT - quote_id, - COALESCE(observed_at, ingested_at) as submitted_at, - payload->>'expected_inventory_delta' as expected_inventory_delta - FROM trade_execution_results - WHERE payload->>'status' = 'submitted' - ORDER BY COALESCE(observed_at, ingested_at) DESC - LIMIT 15000 - `); - const submissions = expectedDeltasRows.rows.map(r => ({ - quote_id: r.quote_id, - submitted_at: r.submitted_at, - expected_inventory_delta: typeof r.expected_inventory_delta === 'string' && r.expected_inventory_delta ? JSON.parse(r.expected_inventory_delta) : r.expected_inventory_delta - })); - - const inventoryRows = await pool.query(` - SELECT - event_id as inventory_id, - observed_at, - payload - FROM intent_inventory_snapshots - ORDER BY COALESCE(observed_at, ingested_at) DESC - LIMIT 50000 - `); - const inventorySnapshots = inventoryRows.rows.map(r => ({ - inventory_id: r.inventory_id, - observed_at: r.observed_at, - asset_balances: r.payload?.asset_balances || {}, - })); - - console.log("Checking for matches...", submissions.length, inventorySnapshots.length); - const records = deriveQuoteOutcomeRecords({ - submissions, - inventorySnapshots, - }); - - const completed = records.filter(r => r.outcome_status === 'completed'); - console.log("Found", completed.length, "completed trades out of", records.length); - process.exit(0); -} - -main().catch(console.error); diff --git a/fix-attributions-fast.mjs b/fix-attributions-fast.mjs deleted file mode 100644 index 4943b46..0000000 --- a/fix-attributions-fast.mjs +++ /dev/null @@ -1,16 +0,0 @@ -import { createPostgresPool, refreshQuoteOutcomes } from './src/lib/postgres.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - console.log("Running refreshQuoteOutcomes with large limits to re-attribute historical trades..."); - const records = await refreshQuoteOutcomes(pool, { - submissionLimit: 15000, - inventoryLimit: 50000, - }); - console.log("Refreshed", records.length, "quotes."); - const completed = records.filter(r => r.outcome_status === 'completed'); - console.log("Found", completed.length, "completed trades!"); - process.exit(0); -} - -main().catch(console.error); diff --git a/fix-attributions.mjs b/fix-attributions.mjs deleted file mode 100644 index 4c1c175..0000000 --- a/fix-attributions.mjs +++ /dev/null @@ -1,16 +0,0 @@ -import { createPostgresPool, refreshQuoteOutcomes } from './src/lib/postgres.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - console.log("Running refreshQuoteOutcomes with large limits to re-attribute historical trades..."); - const records = await refreshQuoteOutcomes(pool, { - submissionLimit: 50000, - inventoryLimit: 600000, - }); - console.log("Refreshed", records.length, "quotes."); - const completed = records.filter(r => r.outcome_status === 'completed'); - console.log("Found", completed.length, "completed trades!"); - process.exit(0); -} - -main().catch(console.error); diff --git a/patch-postgres-payload.mjs b/patch-postgres-payload.mjs deleted file mode 100644 index 7743c79..0000000 --- a/patch-postgres-payload.mjs +++ /dev/null @@ -1,8 +0,0 @@ -import fs from 'fs'; -const file = 'src/lib/postgres.mjs'; -let content = fs.readFileSync(file, 'utf8'); -content = content.replace( - 'SELECT event_id, observed_at, ingested_at, quote_id, payload\n FROM intent_inventory_snapshots', - 'SELECT event_id, observed_at, ingested_at, quote_id, jsonb_build_object(\\'spendable\\', payload->\\'spendable\\', \\'synced_at\\', payload->\\'synced_at\\') AS payload\n FROM intent_inventory_snapshots' -); -fs.writeFileSync(file, content); diff --git a/profile-queries.mjs b/profile-queries.mjs deleted file mode 100644 index 8dcf9b0..0000000 --- a/profile-queries.mjs +++ /dev/null @@ -1,47 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { loadConfig } from './src/lib/config.mjs'; -import * as pg from './src/lib/postgres.mjs'; - -async function main() { - const config = loadConfig(); - const pool = pg.createPostgresPool({ connectionString: config.postgresUrl }); - - const fns = [ - { name: 'loadLatestPortfolioMetric', fn: () => pg.loadLatestPortfolioMetric(pool) }, - { name: 'loadLatestInventorySnapshot', fn: () => pg.loadLatestInventorySnapshot(pool) }, - { name: 'loadLatestMarketPrice', fn: () => pg.loadLatestMarketPrice(pool) }, - { name: 'loadRecentQuotes', fn: () => pg.loadRecentQuotes(pool, { limit: 100 }) }, - { name: 'loadSubmissionSummary', fn: () => pg.loadSubmissionSummary(pool) }, - { name: 'loadSubmissionPage', fn: () => pg.loadSubmissionPage(pool, { page: 1, pageSize: 20 }) }, - { name: 'loadCurrentFundingObservations', fn: () => pg.loadCurrentFundingObservations(pool) }, - { name: 'loadRecentDepositStatuses', fn: () => pg.loadRecentDepositStatuses(pool, { limit: 20 }) }, - { name: 'loadRecentTradeDecisions', fn: () => pg.loadRecentTradeDecisions(pool, { limit: 20 }) }, - { name: 'loadRecentExecuteTradeCommands', fn: () => pg.loadRecentExecuteTradeCommands(pool, { limit: 40 }) }, - { name: 'loadRecentExecutionResults', fn: () => pg.loadRecentExecutionResults(pool, { limit: 40 }) }, - { name: 'loadRecentQuoteOutcomes', fn: () => pg.loadRecentQuoteOutcomes(pool, { limit: 200 }) }, - { name: 'loadSuccessfulQuoteLifecycleEvidence', fn: () => pg.loadSuccessfulQuoteLifecycleEvidence(pool, { limit: 50 }) }, - { name: 'loadQuoteLifecycleRetentionSummary', fn: () => pg.loadQuoteLifecycleRetentionSummary(pool) }, - { name: 'loadRecentQuoteLifecycleRollups', fn: () => pg.loadRecentQuoteLifecycleRollups(pool, { limit: 40 }) }, - { name: 'loadQuoteLifecycleStatistics', fn: () => pg.loadQuoteLifecycleStatistics(pool, { limit: 1000 }) }, - { name: 'loadRecentIntentRequests', fn: () => pg.loadRecentIntentRequests(pool, { limit: 20, btcAsset: config.tradingBtc, eureAsset: config.tradingEure, refreshOutcomes: false }) }, - { name: 'loadRecentAlertTransitions', fn: () => pg.loadRecentAlertTransitions(pool, { limit: 20 }) }, - { name: 'loadRecentEnvironmentStatuses', fn: () => pg.loadRecentEnvironmentStatuses(pool, { limit: 20 }) }, - { name: 'loadAssetCatalogSummary', fn: () => pg.loadAssetCatalogSummary(pool, { limit: 250 }) }, - { name: 'loadPairConfigSummary', fn: () => pg.loadPairConfigSummary(pool) }, - ]; - - for (const { name, fn } of fns) { - const start = Date.now(); - try { - await fn(); - const duration = Date.now() - start; - console.log(`${name}: ${duration}ms`); - } catch (err) { - console.error(`${name}: ERROR ${err.message}`); - } - } - - pool.end(); -} - -main().catch(console.error); diff --git a/src/apps/market-reference-ingest.mjs b/src/apps/market-reference-ingest.mjs index 40be34a..43f63cb 100644 --- a/src/apps/market-reference-ingest.mjs +++ b/src/apps/market-reference-ingest.mjs @@ -451,6 +451,3 @@ async function shutdown() { process.on('SIGINT', shutdown); process.on('SIGTERM', shutdown); -function compact(value) { - return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry != null)); -} diff --git a/src/apps/trade-executor.mjs b/src/apps/trade-executor.mjs index 69a9554..b3bf20d 100644 --- a/src/apps/trade-executor.mjs +++ b/src/apps/trade-executor.mjs @@ -11,7 +11,7 @@ import { buildEventEnvelope, parseEventMessage } from '../core/event-envelope.mj import { createExecutorStateStore } from '../core/executor-state-store.mjs'; import { createIntentRequestController } from '../core/intent-request-controller.mjs'; import { createLogger, serializeError } from '../core/log.mjs'; -import { extendMakerTiming } from '../core/maker-timing.mjs'; +import { extendMakerTiming, roundTimingMs } from '../core/maker-timing.mjs'; import { classifyRelaySubmissionFailure } from '../core/relay-failure-classification.mjs'; import { assertExecuteTradeCommand, @@ -414,11 +414,6 @@ function finishExecutorTiming(timing) { }; } -function roundTimingMs(value) { - const number = Number(value); - return Number.isFinite(number) ? Math.round(number * 1000) / 1000 : null; -} - function recordExecutorQueueDelay(timing, payload) { executorQueueDelayTracker.record({ commandToExecutorMs: timing.command_to_executor_ms, diff --git a/src/core/assets.mjs b/src/core/assets.mjs index 23845c5..a959f80 100644 --- a/src/core/assets.mjs +++ b/src/core/assets.mjs @@ -53,3 +53,9 @@ export function mapToSortedObject(record = {}) { Object.entries(record).sort(([left], [right]) => left.localeCompare(right)), ); } + +export function compactShallow(record = {}) { + return Object.fromEntries( + Object.entries(record).filter(([, value]) => value != null), + ); +} diff --git a/src/core/bridge-assets.mjs b/src/core/bridge-assets.mjs index 8518991..35782e8 100644 --- a/src/core/bridge-assets.mjs +++ b/src/core/bridge-assets.mjs @@ -1,3 +1,5 @@ +import { toIsoTimestamp } from './maker-timing.mjs'; + export function intentsAssetIdFromNearTokenId(nearTokenId) { const normalized = String(nearTokenId || '').trim(); if (!normalized) return null; @@ -57,9 +59,3 @@ export function assetsByChain(assets = []) { } return byChain; } - -function toIsoTimestamp(value) { - if (!value) return null; - const date = new Date(value); - return Number.isNaN(date.getTime()) ? null : date.toISOString(); -} diff --git a/src/core/executor-queue-delay.mjs b/src/core/executor-queue-delay.mjs index e9aaafa..3fa8b7e 100644 --- a/src/core/executor-queue-delay.mjs +++ b/src/core/executor-queue-delay.mjs @@ -1,3 +1,5 @@ +import { percentile, roundTimingMs, toIsoTimestamp } from './maker-timing.mjs'; + export const DEFAULT_EXECUTOR_QUEUE_DELAY_WARNING_MS = 1_000; export const DEFAULT_EXECUTOR_QUEUE_DELAY_CRITICAL_MS = 5_000; export const DEFAULT_EXECUTOR_QUEUE_DELAY_SAMPLE_LIMIT = 50; @@ -97,31 +99,7 @@ export function summarizeExecutorQueueDelaySamples(samples = [], { }; } -function percentile(sortedValues, ratio) { - if (!sortedValues.length) return null; - const index = Math.min( - sortedValues.length - 1, - Math.max(0, Math.ceil(sortedValues.length * ratio) - 1), - ); - return sortedValues[index]; -} - -function roundTimingMs(value) { - const number = Number(value); - return Number.isFinite(number) ? Math.round(number * 1000) / 1000 : null; -} - function positiveInteger(value, fallback) { const parsed = Number(value); return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; } - -function toIsoTimestamp(value) { - if (!value) return null; - if (typeof value === 'string') { - const parsed = Date.parse(value); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null; - } - const date = new Date(value); - return Number.isNaN(date.getTime()) ? null : date.toISOString(); -} diff --git a/src/core/funding-observations.mjs b/src/core/funding-observations.mjs index 1a9acd0..6163ea0 100644 --- a/src/core/funding-observations.mjs +++ b/src/core/funding-observations.mjs @@ -2,7 +2,8 @@ import crypto from 'node:crypto'; import { bigintAmount, mapToSortedObject } from './assets.mjs'; -const CREDITED_BRIDGE_STATUSES = new Set(['COMPLETED', 'CREDITED', 'FINALIZED', 'SETTLED']); +export const CREDITED_STATUSES = new Set(['CREDITED', 'COMPLETED', 'FINALIZED', 'SETTLED']); +export const COMPLETED_WITHDRAWAL_STATUSES = new Set(['COMPLETED', 'FINALIZED', 'SETTLED']); const FAILED_BRIDGE_STATUSES = new Set(['FAILED', 'ERROR', 'REJECTED', 'EXPIRED']); const UNCONFIRMED_STATUS = 'SEEN_UNCONFIRMED'; const CONFIRMED_STATUS = 'SEEN_CONFIRMED'; @@ -50,7 +51,7 @@ export function correlateFundingObservation({ let status = normalizedConfirmations > 0 ? CONFIRMED_STATUS : UNCONFIRMED_STATUS; if (bridgeStatus) { - if (CREDITED_BRIDGE_STATUSES.has(bridgeStatus)) status = CREDITED_STATUS; + if (CREDITED_STATUSES.has(bridgeStatus)) status = CREDITED_STATUS; else if (FAILED_BRIDGE_STATUSES.has(bridgeStatus)) status = FAILED_OR_STUCK_STATUS; else status = CREDIT_PENDING_STATUS; } else if (stuckAfterMs > 0 && ageMs >= stuckAfterMs && normalizedConfirmations > 0) { diff --git a/src/core/intent-request-outcomes.mjs b/src/core/intent-request-outcomes.mjs index f3e75e7..e0e071d 100644 --- a/src/core/intent-request-outcomes.mjs +++ b/src/core/intent-request-outcomes.mjs @@ -1,3 +1,5 @@ +import { toIsoTimestamp } from './maker-timing.mjs'; + const DEFAULT_ATTRIBUTION_WINDOW_MS = 10 * 60 * 1000; const DEFAULT_SETTLEMENT_GRACE_MS = 60 * 1000; @@ -510,12 +512,6 @@ function safeBigInt(value) { return BigInt(String(value)); } -function toIsoTimestamp(value) { - if (!value) return null; - const date = new Date(value); - return Number.isNaN(date.getTime()) ? null : date.toISOString(); -} - function timestampValue(value) { const parsed = Date.parse(value || ''); return Number.isFinite(parsed) ? parsed : NaN; diff --git a/src/core/maker-competitiveness.mjs b/src/core/maker-competitiveness.mjs index cbc06e1..5a7f620 100644 --- a/src/core/maker-competitiveness.mjs +++ b/src/core/maker-competitiveness.mjs @@ -1,4 +1,4 @@ -import { normalizeMakerTiming } from './maker-timing.mjs'; +import { normalizeMakerTiming, percentile } from './maker-timing.mjs'; import { classifyRelaySubmissionFailure } from './relay-failure-classification.mjs'; const LATENCY_FIELDS = [ @@ -270,11 +270,6 @@ function percentiles(values) { }; } -function percentile(sorted, rank) { - const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * rank) - 1); - return sorted[index]; -} - function classifyOutcomeStatus({ row, execution, decision }) { if (decision.decision === 'blocked' && String(decision.decision_reason || '').startsWith('maker_')) { return 'policy_skip'; diff --git a/src/core/maker-timing.mjs b/src/core/maker-timing.mjs index 0103717..7421cfc 100644 --- a/src/core/maker-timing.mjs +++ b/src/core/maker-timing.mjs @@ -135,12 +135,21 @@ function recomputeMakerTiming(timing) { return next; } -function timestampMs(value) { +export function timestampMs(value) { if (!value) return Number.NaN; const parsed = Date.parse(value); return Number.isFinite(parsed) ? parsed : Number.NaN; } +export function percentile(sortedValues, rank) { + if (!sortedValues.length) return null; + const index = Math.min( + sortedValues.length - 1, + Math.max(0, Math.ceil(sortedValues.length * rank) - 1), + ); + return sortedValues[index]; +} + function isRecord(value) { return Boolean(value) && typeof value === 'object' && !Array.isArray(value); } diff --git a/src/core/notification-layer.mjs b/src/core/notification-layer.mjs index e20e8fe..4ebe418 100644 --- a/src/core/notification-layer.mjs +++ b/src/core/notification-layer.mjs @@ -1,7 +1,6 @@ +import { CREDITED_STATUSES, COMPLETED_WITHDRAWAL_STATUSES } from './funding-observations.mjs'; import { formatUnitsDecimal } from './intent-requests.mjs'; -const CREDITED_DEPOSIT_STATUSES = new Set(['CREDITED', 'COMPLETED', 'FINALIZED', 'SETTLED']); -const COMPLETED_WITHDRAWAL_STATUSES = new Set(['COMPLETED', 'FINALIZED', 'SETTLED']); const SETTLED_ATTRIBUTION_STATUSES = new Set(['heuristic_match', 'linked_settlement']); export function buildLiquidityActionNotification({ event, config } = {}) { @@ -9,7 +8,7 @@ export function buildLiquidityActionNotification({ event, config } = {}) { const details = payload.details || {}; const status = normalizeStatus(payload.status || details.status); - if (payload.action_type === 'deposit_status_observed' && CREDITED_DEPOSIT_STATUSES.has(status)) { + if (payload.action_type === 'deposit_status_observed' && CREDITED_STATUSES.has(status)) { if (!details.tx_hash) return null; const asset = assetFor(config, payload.asset_id || details.asset_id); const amount = formatAssetUnits(details.amount || '0', asset); diff --git a/src/core/operator-dashboard.mjs b/src/core/operator-dashboard.mjs index 34d0376..4b1aa4b 100644 --- a/src/core/operator-dashboard.mjs +++ b/src/core/operator-dashboard.mjs @@ -1,6 +1,6 @@ import { unitsToNumber } from './assets.mjs'; import { bridgeDepositObservedAt } from './bridge-assets.mjs'; -import { summarizeFundingObservations } from './funding-observations.mjs'; +import { CREDITED_STATUSES, summarizeFundingObservations } from './funding-observations.mjs'; import { buildMakerCompetitivenessSummary } from './maker-competitiveness.mjs'; import { resolveDashboardRequestAuth } from './operator-dashboard-auth.mjs'; import { buildQuoteLifecycleStatistics } from './quote-lifecycle-statistics.mjs'; @@ -18,7 +18,6 @@ const ALERT_SEVERITY_ORDER = { warning: 2, info: 1, }; -const CREDITED_FUNDING_STATUSES = new Set(['CREDITED', 'COMPLETED', 'FINALIZED', 'SETTLED']); const CONTROL_DEFINITIONS = [ { @@ -947,7 +946,7 @@ function buildFundingSummary({ config, fundingObservations, recentDepositStatuse }; }), credited_deposits: recentFundingActivity - .filter((observation) => CREDITED_FUNDING_STATUSES.has(String(observation?.status || '').toUpperCase())) + .filter((observation) => CREDITED_STATUSES.has(String(observation?.status || '').toUpperCase())) .sort((left, right) => sortTimestamps( fundingActivityTimestamp(right), fundingActivityTimestamp(left), @@ -2682,7 +2681,7 @@ function normalizeLiquidityDepositForUi({ config, deposit, observedAt }) { confirmations: null, first_seen_at: timestamp, last_seen_at: timestamp, - credited_at: CREDITED_FUNDING_STATUSES.has(String(status || '').toUpperCase()) ? timestamp : null, + credited_at: CREDITED_STATUSES.has(String(status || '').toUpperCase()) ? timestamp : null, bridge_created_at: bridgeCreatedAt, }; } diff --git a/src/core/quote-lifecycle-retention.mjs b/src/core/quote-lifecycle-retention.mjs index 117c429..08b9e45 100644 --- a/src/core/quote-lifecycle-retention.mjs +++ b/src/core/quote-lifecycle-retention.mjs @@ -5,6 +5,7 @@ import { notionalBucket, quoteAgeBucket, } from './maker-competitiveness.mjs'; +import { timestampMs, toIsoTimestamp } from './maker-timing.mjs'; import { TERMINAL_SETTLEMENT_ATTRIBUTION_STATUSES } from './quote-outcomes.mjs'; export const QUOTE_LIFECYCLE_RETENTION_POLICY_KEY = 'quote_lifecycle_default'; @@ -447,18 +448,6 @@ function normalizeToken(value) { return String(value || '').trim().toLowerCase(); } -function timestampMs(value) { - if (!value) return NaN; - const parsed = Date.parse(value); - return Number.isFinite(parsed) ? parsed : NaN; -} - -function toIsoTimestamp(value) { - if (!value) return null; - const date = new Date(value); - return Number.isNaN(date.getTime()) ? null : date.toISOString(); -} - function firstTimestamp(values = []) { for (const value of values) { const iso = toIsoTimestamp(value); diff --git a/src/core/quote-lifecycle-statistics.mjs b/src/core/quote-lifecycle-statistics.mjs index 403dbd1..f6740fe 100644 --- a/src/core/quote-lifecycle-statistics.mjs +++ b/src/core/quote-lifecycle-statistics.mjs @@ -1,3 +1,5 @@ +import { toIsoTimestamp } from './maker-timing.mjs'; + export const QUOTE_LIFECYCLE_STATISTIC_GRAINS = Object.freeze([ 'all_time', 'month', @@ -432,12 +434,6 @@ function timestampValue(value) { return Number.isFinite(parsed) ? parsed : -Infinity; } -function toIsoTimestamp(value) { - if (!value) return null; - const date = new Date(value); - return Number.isNaN(date.getTime()) ? null : date.toISOString(); -} - function earliestTimestamp(values = []) { let earliest = null; for (const value of values) { diff --git a/src/core/quote-outcomes.mjs b/src/core/quote-outcomes.mjs index 8b23f66..59fbb6d 100644 --- a/src/core/quote-outcomes.mjs +++ b/src/core/quote-outcomes.mjs @@ -1,4 +1,4 @@ -import { extendMakerTiming } from './maker-timing.mjs'; +import { extendMakerTiming, toIsoTimestamp } from './maker-timing.mjs'; const DEFAULT_ATTRIBUTION_WINDOW_MS = 10 * 60 * 1000; const DEFAULT_SETTLEMENT_GRACE_MS = 60 * 1000; @@ -573,12 +573,6 @@ function safeBigInt(value) { return BigInt(String(value)); } -function toIsoTimestamp(value) { - if (!value) return null; - const date = new Date(value); - return Number.isNaN(date.getTime()) ? null : date.toISOString(); -} - function timestampValue(value) { const parsed = Date.parse(value || ''); return Number.isFinite(parsed) ? parsed : NaN; diff --git a/src/core/strategy.mjs b/src/core/strategy.mjs index dd3852a..97e8f21 100644 --- a/src/core/strategy.mjs +++ b/src/core/strategy.mjs @@ -3,6 +3,7 @@ import crypto from 'node:crypto'; import { bigintAmount, classifyPairDirection, + compactShallow, formatNumber, numberToUnits, pairKey, @@ -84,7 +85,7 @@ export function evaluateTradeOpportunity({ maker_receive_asset_id: payload.asset_in, maker_send_symbol: pairRuntime.assetOut?.symbol || null, maker_receive_symbol: pairRuntime.assetIn?.symbol || null, - assumptions: compact({ + assumptions: compactShallow({ eure_per_eur: pairRuntime.priceRoute?.source === 'btc_eur_reference' ? '1' : null, usdc_per_usd: pairRuntime.priceRoute?.source === 'btc_usdc_reference' ? '1' : null, price_route_source: pairRuntime.priceRoute?.source || null, @@ -552,19 +553,13 @@ function finalizeQuote({ return { ok: true, details: reasonBase, - quoteOutput: compact({ + quoteOutput: compactShallow({ amount_in: proposedAmountIn, amount_out: proposedAmountOut, }), }; } -function compact(value) { - return Object.fromEntries( - Object.entries(value).filter(([, entry]) => entry != null), - ); -} - function withReason(decision, reason) { return { ...decision, @@ -605,7 +600,7 @@ function buildMakerTradeTerms({ const sendAmount = demand.request_kind === 'exact_in' ? quoteOutput?.amount_out || proposedAmountOut : demand.amount_out; - return compact({ + return compactShallow({ maker_send_asset_id: demand.asset_out, maker_receive_asset_id: demand.asset_in, maker_send_amount_units: sendAmount == null ? null : String(sendAmount), diff --git a/src/core/verifier-salt-cache.mjs b/src/core/verifier-salt-cache.mjs index 6f48a71..1127418 100644 --- a/src/core/verifier-salt-cache.mjs +++ b/src/core/verifier-salt-cache.mjs @@ -1,3 +1,5 @@ +import { toIsoTimestamp } from './maker-timing.mjs'; + const DEFAULT_MAX_AGE_MS = 500; const DEFAULT_REFRESH_INTERVAL_MS = 250; const ERROR_LOG_SUPPRESSION_MS = 30_000; @@ -229,8 +231,3 @@ function timestampMs(value) { if (value instanceof Date) return value.getTime(); return Date.parse(value); } - -function toIsoTimestamp(value) { - const parsed = timestampMs(value); - return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null; -} diff --git a/src/venues/near-intents/verifier-client.mjs b/src/venues/near-intents/verifier-client.mjs index 0cd2fc4..2df77af 100644 --- a/src/venues/near-intents/verifier-client.mjs +++ b/src/venues/near-intents/verifier-client.mjs @@ -1,5 +1,6 @@ import { Account, JsonRpcProvider, KeyPair, teraToGas } from 'near-api-js'; +import { compactShallow } from '../../core/assets.mjs'; import { postJson } from '../../lib/http.mjs'; export function createVerifierClient({ @@ -54,7 +55,7 @@ export function createVerifierClient({ return operatorAccount.callFunctionRaw({ contractId: verifierContract, methodName: 'ft_withdraw', - args: compact({ + args: compactShallow({ token, receiver_id: receiverId, amount: String(amount), @@ -129,8 +130,3 @@ async function callView(provider, accountId, methodName, args) { return text ? JSON.parse(text) : null; } -function compact(record) { - return Object.fromEntries( - Object.entries(record).filter(([, value]) => value != null), - ); -} diff --git a/test-all-completed.mjs b/test-all-completed.mjs deleted file mode 100644 index 7e11e9b..0000000 --- a/test-all-completed.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - const rows = await pool.query(` - SELECT count(*) FROM quote_outcome_attributions WHERE outcome_status = 'completed'; - `); - console.log("Total completed in DB:", rows.rows[0].count); - process.exit(0); -} -main().catch(console.error); diff --git a/test-dashboard-fetch.mjs b/test-dashboard-fetch.mjs deleted file mode 100644 index 40f661c..0000000 --- a/test-dashboard-fetch.mjs +++ /dev/null @@ -1,20 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { refreshQuoteOutcomes } from './src/lib/postgres.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - - console.log("Running refreshQuoteOutcomes..."); - try { - const records = await refreshQuoteOutcomes(pool, { - btcAsset: { assetId: 'nep141:nbtc.bridge.near' }, - eureAsset: { assetId: 'nep141:eure.omft.near' } - }); - console.log("Records length:", records.length); - console.log("Completed:", records.filter(r => r.outcome_status === 'completed').length); - } catch (err) { - console.error("Error:", err); - } - process.exit(0); -} -main().catch(console.error); diff --git a/test-dashboard-speed.mjs b/test-dashboard-speed.mjs deleted file mode 100644 index 92ad755..0000000 --- a/test-dashboard-speed.mjs +++ /dev/null @@ -1,37 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import * as db from './src/lib/postgres.mjs'; - -async function measure(name, fn) { - const start = Date.now(); - try { - await fn(); - console.log(`[OK] ${name}: ${Date.now() - start}ms`); - } catch (e) { - console.log(`[ERR] ${name}: ${Date.now() - start}ms`, e.message); - } -} - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - await Promise.all([ - measure('loadLatestPortfolioMetric', () => db.loadLatestPortfolioMetric(pool)), - measure('loadLatestInventorySnapshot', () => db.loadLatestInventorySnapshot(pool)), - measure('loadLatestMarketPrice', () => db.loadLatestMarketPrice(pool)), - measure('loadRecentQuotes', () => db.loadRecentQuotes(pool, { limit: 100 })), - measure('loadSubmissionSummary', () => db.loadSubmissionSummary(pool)), - measure('loadSubmissionPage', () => db.loadSubmissionPage(pool, { page: 1, pageSize: 50 })), - measure('loadCurrentFundingObservations', () => db.loadCurrentFundingObservations(pool)), - measure('loadRecentDepositStatuses', () => db.loadRecentDepositStatuses(pool, { limit: 20 })), - measure('loadRecentTradeDecisions', () => db.loadRecentTradeDecisions(pool, { limit: 20 })), - measure('loadRecentExecuteTradeCommands', () => db.loadRecentExecuteTradeCommands(pool, { limit: 40 })), - measure('loadRecentExecutionResults', () => db.loadRecentExecutionResults(pool, { limit: 40 })), - measure('loadRecentQuoteOutcomes', () => db.loadRecentQuoteOutcomes(pool, { limit: 200 })), - measure('loadSuccessfulQuoteLifecycleEvidence', () => db.loadSuccessfulQuoteLifecycleEvidence(pool, { limit: 50 })), - measure('loadQuoteLifecycleRetentionSummary', () => db.loadQuoteLifecycleRetentionSummary(pool)), - measure('loadRecentQuoteLifecycleRollups', () => db.loadRecentQuoteLifecycleRollups(pool, { limit: 40 })), - measure('loadQuoteLifecycleStatistics', () => db.loadQuoteLifecycleStatistics(pool, { limit: 1000 })), - measure('loadRecentIntentRequests', () => db.loadRecentIntentRequests(pool, { limit: 50 })), - ]); - process.exit(0); -} -main().catch(console.error); diff --git a/test-deltas.mjs b/test-deltas.mjs deleted file mode 100644 index 4d79ae6..0000000 --- a/test-deltas.mjs +++ /dev/null @@ -1,38 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - const inventoryRows = await pool.query(` - SELECT event_id as inventory_id, observed_at, payload - FROM intent_inventory_snapshots - ORDER BY COALESCE(observed_at, ingested_at) DESC - LIMIT 2000 - `); - const snapshots = inventoryRows.rows.map(r => ({ - inventory_id: r.inventory_id, - observed_at: r.observed_at, - spendable: r.payload?.spendable || {}, - })).sort((a,b) => new Date(a.observed_at) - new Date(b.observed_at)); - - const activeAssetIds = Object.keys(snapshots[0].spendable); - const deltas = []; - for (let i = 1; i < snapshots.length; i++) { - const prev = snapshots[i-1].spendable; - const curr = snapshots[i].spendable; - const delta = {}; - for (const a of activeAssetIds) { - const p = BigInt(prev[a] || 0); - const c = BigInt(curr[a] || 0); - if (c - p !== 0n) delta[a] = (c - p).toString(); - } - if (Object.keys(delta).length > 0) { - deltas.push({ observed_at: snapshots[i].observed_at, delta }); - } - } - console.log("Total deltas:", deltas.length); - for (let i = 0; i < Math.min(5, deltas.length); i++) { - console.log(deltas[i].observed_at, Object.entries(deltas[i].delta).filter(x => x[1] !== '0')); - } - process.exit(0); -} -main().catch(console.error); diff --git a/test-matches-all.mjs b/test-matches-all.mjs deleted file mode 100644 index b15b863..0000000 --- a/test-matches-all.mjs +++ /dev/null @@ -1,43 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { deriveQuoteOutcomeRecords } from './src/core/quote-outcomes.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - - const submissionsResult = await pool.query(` - SELECT event_id, observed_at, ingested_at, quote_id, payload - FROM trade_execution_results - WHERE payload->>'status' = 'submitted' - `); - const submissions = submissionsResult.rows; - - const [commandsResult, decisionsResult, inventoryResult] = await Promise.all([ - pool.query(`SELECT event_id, observed_at, ingested_at, quote_id, payload FROM execute_trade_commands`), - pool.query(`SELECT event_id, observed_at, ingested_at, quote_id, payload FROM trade_decisions`), - pool.query(` - SELECT event_id, observed_at, ingested_at, payload - FROM intent_inventory_snapshots - ORDER BY COALESCE(observed_at, ingested_at) DESC - LIMIT 15000 - `), - ]); - - const btcAsset = { assetId: 'nep141:nbtc.bridge.near' }; - const eureAsset = { assetId: 'nep141:eure.omft.near' }; - - console.log("Running derive..."); - const records = deriveQuoteOutcomeRecords({ - submissions, - commands: commandsResult.rows, - decisions: decisionsResult.rows, - inventorySnapshots: inventoryResult.rows, - btcAsset, - eureAsset, - now: new Date().toISOString() - }); - - const completed = records.filter(r => r.outcome_status === 'completed'); - console.log("Found completed:", completed.length); - process.exit(0); -} -main().catch(console.error); diff --git a/test-matches-remote-5000.mjs b/test-matches-remote-5000.mjs deleted file mode 100644 index 25973ba..0000000 --- a/test-matches-remote-5000.mjs +++ /dev/null @@ -1,45 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { deriveQuoteOutcomeRecords } from './src/core/quote-outcomes.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - - const submissionsResult = await pool.query(` - SELECT event_id, observed_at, ingested_at, quote_id, payload - FROM trade_execution_results - WHERE payload->>'status' = 'submitted' - `); - const submissions = submissionsResult.rows; - - const [commandsResult, decisionsResult, inventoryResult] = await Promise.all([ - pool.query(`SELECT event_id, observed_at, ingested_at, quote_id, payload FROM execute_trade_commands`), - pool.query(`SELECT event_id, observed_at, ingested_at, quote_id, payload FROM trade_decisions`), - pool.query(` - SELECT event_id, observed_at, ingested_at, payload - FROM intent_inventory_snapshots - ORDER BY COALESCE(observed_at, ingested_at) DESC - LIMIT 5000 - `), - ]); - - console.log("Memory before derive:", process.memoryUsage().heapUsed / 1024 / 1024, "MB"); - - const btcAsset = { assetId: 'nep141:nbtc.bridge.near' }; - const eureAsset = { assetId: 'nep141:eure.omft.near' }; - - console.log("Running derive..."); - const records = deriveQuoteOutcomeRecords({ - submissions, - commands: commandsResult.rows, - decisions: decisionsResult.rows, - inventorySnapshots: inventoryResult.rows, - btcAsset, - eureAsset, - now: new Date().toISOString() - }); - - const completed = records.filter(r => r.outcome_status === 'completed'); - console.log("Found completed:", completed.length); - process.exit(0); -} -main().catch(console.error); diff --git a/test-matches-remote.mjs b/test-matches-remote.mjs deleted file mode 100644 index b15b863..0000000 --- a/test-matches-remote.mjs +++ /dev/null @@ -1,43 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { deriveQuoteOutcomeRecords } from './src/core/quote-outcomes.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - - const submissionsResult = await pool.query(` - SELECT event_id, observed_at, ingested_at, quote_id, payload - FROM trade_execution_results - WHERE payload->>'status' = 'submitted' - `); - const submissions = submissionsResult.rows; - - const [commandsResult, decisionsResult, inventoryResult] = await Promise.all([ - pool.query(`SELECT event_id, observed_at, ingested_at, quote_id, payload FROM execute_trade_commands`), - pool.query(`SELECT event_id, observed_at, ingested_at, quote_id, payload FROM trade_decisions`), - pool.query(` - SELECT event_id, observed_at, ingested_at, payload - FROM intent_inventory_snapshots - ORDER BY COALESCE(observed_at, ingested_at) DESC - LIMIT 15000 - `), - ]); - - const btcAsset = { assetId: 'nep141:nbtc.bridge.near' }; - const eureAsset = { assetId: 'nep141:eure.omft.near' }; - - console.log("Running derive..."); - const records = deriveQuoteOutcomeRecords({ - submissions, - commands: commandsResult.rows, - decisions: decisionsResult.rows, - inventorySnapshots: inventoryResult.rows, - btcAsset, - eureAsset, - now: new Date().toISOString() - }); - - const completed = records.filter(r => r.outcome_status === 'completed'); - console.log("Found completed:", completed.length); - process.exit(0); -} -main().catch(console.error); diff --git a/test-memory-2.mjs b/test-memory-2.mjs deleted file mode 100644 index f4460a2..0000000 --- a/test-memory-2.mjs +++ /dev/null @@ -1,15 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - -async function main() { - console.log("Memory before:", process.memoryUsage().heapUsed / 1024 / 1024, "MB"); - const result = await pool.query(` - SELECT payload - FROM trade_execution_results - WHERE payload->>'status' = 'submitted' - LIMIT 1000 - `); - console.log("Memory after fetch:", process.memoryUsage().heapUsed / 1024 / 1024, "MB"); - process.exit(0); -} -main().catch(console.error); diff --git a/test-memory-3.mjs b/test-memory-3.mjs deleted file mode 100644 index 8febee5..0000000 --- a/test-memory-3.mjs +++ /dev/null @@ -1,13 +0,0 @@ -import { createPostgresPool, refreshQuoteOutcomes } from './src/lib/postgres.mjs'; -const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - -async function main() { - console.log("Memory before:", process.memoryUsage().heapUsed / 1024 / 1024, "MB"); - const records = await refreshQuoteOutcomes(pool, { - btcAsset: { assetId: 'nep141:nbtc.bridge.near' }, - eureAsset: { assetId: 'nep141:eure.omft.near' } - }); - console.log("Memory after:", process.memoryUsage().heapUsed / 1024 / 1024, "MB"); - process.exit(0); -} -main().catch(console.error); diff --git a/test-memory.mjs b/test-memory.mjs deleted file mode 100644 index 30f75fd..0000000 --- a/test-memory.mjs +++ /dev/null @@ -1,20 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - -async function main() { - console.log("Memory before:", process.memoryUsage().heapUsed / 1024 / 1024, "MB"); - const result = await pool.query(` - SELECT event_id, observed_at, ingested_at, quote_id, jsonb_build_object('spendable', payload->'spendable', 'synced_at', payload->'synced_at') AS payload - FROM ( - SELECT event_id, observed_at, ingested_at, quote_id, payload - FROM intent_inventory_snapshots - ORDER BY COALESCE(observed_at, ingested_at) DESC - LIMIT 50000 - ) recent_inventory_snapshots - ORDER BY COALESCE(observed_at, ingested_at) ASC - `); - console.log("Memory after fetch:", process.memoryUsage().heapUsed / 1024 / 1024, "MB"); - console.log("Rows fetched:", result.rows.length); - process.exit(0); -} -main().catch(console.error); diff --git a/test-outcomes-2.mjs b/test-outcomes-2.mjs deleted file mode 100644 index b53dc44..0000000 --- a/test-outcomes-2.mjs +++ /dev/null @@ -1,15 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { refreshQuoteOutcomes } from './src/lib/postgres.mjs'; -const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); -async function main() { - console.log("Starting..."); - try { - const records = await refreshQuoteOutcomes(pool, { - btcAsset: { assetId: 'nep141:nbtc.bridge.near' }, - eureAsset: { assetId: 'nep141:eure.omft.near' } - }); - console.log("Returned:", records.length); - } catch(e) { console.error("Error:", e); } - process.exit(0); -} -main(); diff --git a/test-outcomes-3.mjs b/test-outcomes-3.mjs deleted file mode 100644 index 1f62bd9..0000000 --- a/test-outcomes-3.mjs +++ /dev/null @@ -1,17 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { refreshQuoteOutcomes } from './src/lib/postgres.mjs'; -const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); -async function main() { - console.log("Starting debug run..."); - try { - const t0 = Date.now(); - console.log("Calling pool.query inside refreshQuoteOutcomes..."); - const records = await refreshQuoteOutcomes(pool, { - btcAsset: { assetId: 'nep141:nbtc.bridge.near' }, - eureAsset: { assetId: 'nep141:eure.omft.near' } - }); - console.log(`Returned: ${records.length} in ${Date.now() - t0}ms`); - } catch(e) { console.error("Error:", e); } - process.exit(0); -} -main(); diff --git a/test-outcomes-4.mjs b/test-outcomes-4.mjs deleted file mode 100644 index 28a1240..0000000 --- a/test-outcomes-4.mjs +++ /dev/null @@ -1,20 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { refreshQuoteOutcomes } from './src/lib/postgres.mjs'; -import { deriveInventoryDeltas } from './src/core/quote-outcomes.mjs'; - -const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - -async function main() { - console.log("Starting debug run..."); - try { - const t0 = Date.now(); - const records = await refreshQuoteOutcomes(pool, { - btcAsset: { assetId: 'nep141:nbtc.bridge.near' }, - eureAsset: { assetId: 'nep141:eure.omft.near' } - }); - console.log(`Returned: ${records.length} in ${Date.now() - t0}ms`); - } catch(e) { console.error("Error:", e); } - process.exit(0); -} - -main(); diff --git a/test-outcomes-5.mjs b/test-outcomes-5.mjs deleted file mode 100644 index e3a9bea..0000000 --- a/test-outcomes-5.mjs +++ /dev/null @@ -1,23 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { refreshQuoteOutcomes } from './src/lib/postgres.mjs'; - -const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - -async function main() { - console.log("Starting debug run..."); - try { - const records = await refreshQuoteOutcomes(pool, { - btcAsset: { assetId: 'nep141:nbtc.bridge.near' }, - eureAsset: { assetId: 'nep141:eure.omft.near' } - }); - console.log("Returned records:", records.length); - let completed = 0; - for (const r of records) { - if (r.outcome_status === 'completed') completed++; - } - console.log("Completed outcomes:", completed); - } catch(e) { console.error("Error:", e); } - process.exit(0); -} - -main(); diff --git a/test-outcomes.mjs b/test-outcomes.mjs deleted file mode 100644 index 87b8380..0000000 --- a/test-outcomes.mjs +++ /dev/null @@ -1,16 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; -import { refreshQuoteOutcomes } from './src/lib/postgres.mjs'; -const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); -async function main() { - console.log("Starting refreshQuoteOutcomes..."); - try { - const records = await refreshQuoteOutcomes(pool, { - btcAsset: { assetId: 'nep141:nbtc.bridge.near' }, - eureAsset: { assetId: 'nep141:eure.omft.near' } - }); - console.log("Returned records:", records.length); - console.log("Completed:", records.filter(r => r.outcome_status === 'completed').length); - } catch(e) { console.error("Error:", e); } - process.exit(0); -} -main(); diff --git a/test-perf.mjs b/test-perf.mjs deleted file mode 100644 index f767c94..0000000 --- a/test-perf.mjs +++ /dev/null @@ -1,14 +0,0 @@ -import { deriveQuoteOutcomeRecords } from './src/core/quote-outcomes.mjs'; -const submissions = Array.from({length: 2000}).map((_, i) => ({ - quote_id: 'q' + i, - submitted_at: new Date().toISOString(), - expected_inventory_delta: { asset1: 10n } -})); -const inventorySnapshots = Array.from({length: 5000}).map((_, i) => ({ - inventory_id: 'i' + i, - observed_at: new Date(Date.now() + i * 1000).toISOString(), - asset_balances: { asset1: BigInt(i * 10) } -})); -console.time('derive'); -deriveQuoteOutcomeRecords({ submissions, inventorySnapshots }); -console.timeEnd('derive'); diff --git a/test-recent.mjs b/test-recent.mjs deleted file mode 100644 index 1fa4781..0000000 --- a/test-recent.mjs +++ /dev/null @@ -1,14 +0,0 @@ -import { createPostgresPool } from './src/lib/postgres.mjs'; - -async function main() { - const pool = createPostgresPool({ connectionString: process.env.POSTGRES_URL }); - const inventoryRows = await pool.query(` - SELECT event_id as inventory_id, observed_at, payload - FROM intent_inventory_snapshots - ORDER BY COALESCE(observed_at, ingested_at) DESC - LIMIT 1 - `); - console.log("Most recent snapshot:", inventoryRows.rows[0].observed_at); - process.exit(0); -} -main().catch(console.error);