diff --git a/.env.example b/.env.example index 4fe5019..763db8e 100644 --- a/.env.example +++ b/.env.example @@ -1,26 +1,79 @@ -# Local dev / container runtime values +# Core NEAR Intents runtime NEAR_INTENTS_API_KEY=replace_me +NEAR_INTENTS_ACCOUNT_ID=solver.near +NEAR_INTENTS_SIGNER_PRIVATE_KEY=ed25519:replace_me NEAR_INTENTS_WS_URL=wss://solver-relay-v2.chaindefuser.com/ws +NEAR_INTENTS_RPC_URL=https://solver-relay-v2.chaindefuser.com/rpc +NEAR_INTENTS_BRIDGE_RPC_URL=https://bridge.chaindefuser.com/rpc +NEAR_INTENTS_VERIFIER_CONTRACT=intents.near +NEAR_RPC_URL=https://rpc.fastnear.com NEAR_INTENTS_PAIR_FILTER=nep141:btc.omft.near->nep141:gnosis-0x420ca0f9b9b604ce0fd9c18ef134c705e5fa3430.omft.near + +# Active pair metadata +TRADING_BTC_ASSET_ID=nep141:btc.omft.near +TRADING_BTC_SYMBOL=BTC +TRADING_BTC_DECIMALS=8 +TRADING_BTC_CHAIN=btc:mainnet +TRADING_EURE_ASSET_ID=nep141:gnosis-0x420ca0f9b9b604ce0fd9c18ef134c705e5fa3430.omft.near +TRADING_EURE_SYMBOL=EURe +TRADING_EURE_DECIMALS=18 +TRADING_EURE_CHAIN=eth:100 + +# Control APIs NEAR_INTENTS_CONTROL_API_ENABLED=true NEAR_INTENTS_CONTROL_HOST=0.0.0.0 NEAR_INTENTS_CONTROL_PORT=8081 +MARKET_REFERENCE_CONTROL_HOST=0.0.0.0 +MARKET_REFERENCE_CONTROL_PORT=8082 +INVENTORY_SYNC_CONTROL_HOST=0.0.0.0 +INVENTORY_SYNC_CONTROL_PORT=8083 +LIQUIDITY_MANAGER_CONTROL_HOST=0.0.0.0 +LIQUIDITY_MANAGER_CONTROL_PORT=8084 +HISTORY_WRITER_CONTROL_HOST=0.0.0.0 +HISTORY_WRITER_CONTROL_PORT=8085 +STRATEGY_ENGINE_CONTROL_HOST=0.0.0.0 +STRATEGY_ENGINE_CONTROL_PORT=8086 +TRADE_EXECUTOR_CONTROL_HOST=0.0.0.0 +TRADE_EXECUTOR_CONTROL_PORT=8087 + +# Kafka backbone KAFKA_BROKERS=redpanda:9092 KAFKA_CLIENT_ID=unrip KAFKA_TOPIC_RAW_NEAR_INTENTS_QUOTE=raw.near_intents.quote KAFKA_TOPIC_NORM_SWAP_DEMAND=norm.swap_demand +KAFKA_TOPIC_REF_MARKET_PRICE=ref.market_price +KAFKA_TOPIC_STATE_INTENT_INVENTORY=state.intent_inventory +KAFKA_TOPIC_OPS_LIQUIDITY_ACTION=ops.liquidity_action +KAFKA_TOPIC_DECISION_TRADE_DECISION=decision.trade_decision KAFKA_TOPIC_CMD_EXECUTE_TRADE=cmd.execute_trade KAFKA_TOPIC_EXEC_TRADE_RESULT=exec.trade_result -KAFKA_CONSUMER_GROUP_DUMMY=dummy-reactor-v1 -KAFKA_CONSUMER_GROUP_EXECUTOR=dummy-executor-v1 -EXECUTOR_STATE_DIR=/var/lib/unrip/executor-state +KAFKA_CONSUMER_GROUP_HISTORY=history-writer-v1 +KAFKA_CONSUMER_GROUP_INVENTORY=inventory-sync-v1 +KAFKA_CONSUMER_GROUP_STRATEGY=strategy-engine-v1 +KAFKA_CONSUMER_GROUP_EXECUTOR=trade-executor-v1 -# Platform bootstrap values live in the separate infra/platform repo, not in -# this application repo. In the current local split that repo is `../unrip3`. -# Configure and run bootstrap there, then deploy this repo using the app-side -# workflow described in `docs/deployment.md`. -# -# Future k3s deployment should source the app values from Kubernetes Secret/ConfigMap. -# This repo expects app-side cluster secrets such as: -# - `unrip-secrets` for `NEAR_INTENTS_API_KEY` -# - `unrip-registry-creds` for image pulls and in-cluster Kaniko builds +# PostgreSQL durable history store +POSTGRES_URL=postgresql://unrip:unrip@postgres:5432/unrip + +# Service state +EXECUTOR_STATE_DIR=/var/lib/unrip/executor-state +LIQUIDITY_STATE_DIR=/var/lib/unrip/liquidity-state + +# Pricing and sync cadence +MARKET_REFERENCE_REFRESH_MS=5000 +MARKET_REFERENCE_COINGECKO_REFRESH_MS=15000 +MARKET_REFERENCE_MAX_AGE_MS=30000 +MARKET_REFERENCE_KRAKEN_TICKER_URL=https://api.kraken.com/0/public/Ticker?pair=XBTEUR +MARKET_REFERENCE_COINGECKO_URL=https://api.coingecko.com/api/v3/simple/price?ids=bitcoin&vs_currencies=eur +INVENTORY_SYNC_REFRESH_MS=15000 +LIQUIDITY_REFRESH_MS=30000 + +# Strategy and executor safety defaults +STRATEGY_GROSS_THRESHOLD_PCT=2 +STRATEGY_INITIAL_ARMED=false +STRATEGY_MAX_NOTIONAL_EURE=5 +STRATEGY_PRICE_MAX_AGE_MS=30000 +STRATEGY_INVENTORY_MAX_AGE_MS=30000 +EXECUTOR_INITIAL_ARMED=false +EXECUTOR_RESPONSE_TIMEOUT_MS=10000 +LIQUIDITY_WITHDRAWALS_FROZEN=true diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index f57fa57..3b6b387 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -14,7 +14,7 @@ jobs: REGISTRY_HOST: ${{ vars.REGISTRY_HOST }} PROJECT_NAME: ${{ vars.PROJECT_NAME || 'unrip' }} PROJECT_NAMESPACE: ${{ vars.PROJECT_NAMESPACE || vars.PROJECT_NAME || 'unrip' }} - PROJECT_DEPLOYMENTS: ${{ vars.PROJECT_DEPLOYMENTS || 'near-intents-ingest,dummy-reactor,dummy-executor,dummy-consumer' }} + PROJECT_DEPLOYMENTS: ${{ vars.PROJECT_DEPLOYMENTS || 'near-intents-ingest,market-reference-ingest,liquidity-manager,inventory-sync,history-writer,strategy-engine,trade-executor' }} PROJECT_REGISTRY_SECRET_NAME: ${{ vars.PROJECT_REGISTRY_SECRET_NAME || format('{0}-registry-creds', vars.PROJECT_NAME || 'unrip') }} REPO_CLONE_URL: ${{ github.server_url }}/${{ github.repository }}.git steps: diff --git a/Dockerfile b/Dockerfile index 3451298..2a509b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,4 @@ RUN npm ci --omit=dev COPY . . ENV NODE_ENV=production -CMD ["node", "src/apps/dummy-consumer.mjs"] +CMD ["node", "src/apps/near-intents-ingest.mjs"] diff --git a/README.md b/README.md index fd870a5..13b4a50 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,16 @@ Useful commands: ```bash docker compose ps docker compose logs -f -docker compose logs -f near-intents-ingest dummy-reactor dummy-executor dummy-consumer +docker compose logs -f \ + near-intents-ingest market-reference-ingest liquidity-manager \ + inventory-sync history-writer strategy-engine trade-executor npm run near-intents:ingest -npm run dummy-reactor -npm run dummy-executor -npm run dummy-consumer +npm run market-reference:ingest +npm run liquidity:manager +npm run inventory:sync +npm run history:writer +npm run strategy:engine +npm run trade:executor ``` ## App image @@ -60,6 +65,7 @@ Forgejo runner, registry, and other platform services live in the separate platform repo. See `docs/deployment.md` for the full operator path. +See `docs/operator-runbook.md` for the live turn control and arming sequence. ### One-time app bootstrap @@ -104,9 +110,12 @@ git push forgejo main ```bash KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip get deploy,pods,job KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip rollout status deploy/near-intents-ingest -KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip rollout status deploy/dummy-reactor -KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip rollout status deploy/dummy-executor -KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip rollout status deploy/dummy-consumer +KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip rollout status deploy/market-reference-ingest +KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip rollout status deploy/liquidity-manager +KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip rollout status deploy/inventory-sync +KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip rollout status deploy/history-writer +KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip rollout status deploy/strategy-engine +KUBECONFIG=../unrip3/.state/hetzner/kubeconfig.yaml kubectl -n unrip rollout status deploy/trade-executor ``` ### Auxiliary ops scripts @@ -117,37 +126,51 @@ bootstrap: `../unrip3/.state/hetzner/kubeconfig.yaml`. Override with `scripts/ops/deployment_status.py` - Shows current deployment readiness, pod uptime, restart counts, and mounted storage usage. +- Outputs JSON by default so it can be piped directly into `jq`. - By default it shows the live deployment pods only. - Use `--include-completed` to include completed Job pods. - Use `--include-rootfs` if you also want a probe of `/` for pods without PVC-backed mounts. +- Use `--output table` for the human-readable table view. ```bash python3 scripts/ops/deployment_status.py +python3 scripts/ops/deployment_status.py | jq '.pods[] | {name, uptime}' python3 scripts/ops/deployment_status.py --include-completed +python3 scripts/ops/deployment_status.py --output table ``` `scripts/ops/redpanda_storage.py` - Shows how much data Redpanda is currently storing for the unrip topics. -- Prints per-topic local bytes, total bytes, segment counts, and the overall Redpanda data-path usage. +- Outputs JSON by default so it can be piped directly into `jq`. +- Includes per-topic local bytes, total bytes, segment counts, and the overall Redpanda data-path usage. - Use `--all-topics` to inspect every visible topic, or `--topic` multiple times for a subset. +- Use `--output table` for the human-readable table view. ```bash python3 scripts/ops/redpanda_storage.py +python3 scripts/ops/redpanda_storage.py | jq '.totals' python3 scripts/ops/redpanda_storage.py --all-topics python3 scripts/ops/redpanda_storage.py --topic raw.near_intents.quote --topic norm.swap_demand +python3 scripts/ops/redpanda_storage.py --output table ``` `scripts/ops/live_near_intents.py` -- Live-inspects the raw NEAR quote stream entering Redpanda. -- Defaults to the configured raw topic, `--offset end`, and an unbounded live tail. -- Use `--num` for a bounded sample, `--offset start` to read from the beginning, `--value-only` for payload-only output, or `--rpk-json` for the full record metadata emitted by `rpk`. +- Reads the raw NEAR quote stream entering Redpanda. +- Outputs clean JSON by default. Bounded reads return a JSON array that can be piped directly into `jq`. +- `--num N` means "consume N records starting from the chosen offset". With the default `--offset end`, that means "wait for N new records from now". +- Use `--last N` when you want the most recent retained N records. +- Use `--offset start` to read from the beginning of retained history. +- Use `--output text` for the human-readable stream view or `--output jsonl` for one JSON object per line. +- Use `--value-only` to emit only decoded record values in JSON mode. - Use `--timeout` when you want the script to stop automatically. ```bash -python3 scripts/ops/live_near_intents.py +python3 scripts/ops/live_near_intents.py --last 10 +python3 scripts/ops/live_near_intents.py --last 10 | jq '.[].value.payload.message.quote_id' python3 scripts/ops/live_near_intents.py --num 10 --offset start -python3 scripts/ops/live_near_intents.py --value-only --timeout 30 -python3 scripts/ops/live_near_intents.py --rpk-json --num 5 +python3 scripts/ops/live_near_intents.py --num 10 --timeout 30 +python3 scripts/ops/live_near_intents.py --value-only --last 5 +python3 scripts/ops/live_near_intents.py --output text ``` ### Near Intents control API @@ -185,3 +208,109 @@ Reset the runtime filter back to the configured env/file/default state: ```bash curl -s -X POST http://127.0.0.1:8081/pair-filter/reset ``` + +## Agent workflow + +This repo now carries a small tracked workflow layer for Codex or other agents. +It is meant to create pressure toward real product progress without introducing +heavy orchestration. + +The key files are: + +- `AGENTS.md` — hard rules for agent behavior in this repo +- `THESIS.md` — stable product intent and approval boundaries +- `PROOF.md` — the active implementation proof +- `IMPLEMENTATION.md` — the current implementation turn +- `research/ACTIVE.md` — the active research charter +- `BACKLOG.md` — parked ideas, bugs, and future turn candidates +- `ARCHIVE.md` — index of archived turns and planning events +- `workflow/REVIEW_PROMPT.md` — adversarial review prompt for a separate review-only run + +Two important rules shape the workflow: + +- quote collection and analytics are first-class from day one +- backlog items do not automatically become active implementation without an explicit turn-opening step + +### Install the tracked git hook + +Install the tracked hook path once per clone: + +```bash +bash scripts/workflow/install_hooks.sh +``` + +That sets `core.hooksPath` to `.githooks`. + +The commit hook rejects non-merge commits unless the commit message body +contains: + +- `Proof: ...` +- `Assumptions: ...` +- `Still fake: ...` + +### Workflow scripts + +`scripts/workflow/add_backlog.py` +- Append a new idea, bug, research item, or ops task to `BACKLOG.md`. +- Prints the created stable backlog ID. + +```bash +python3 scripts/workflow/add_backlog.py --lane implementation --summary "Durable sink for normalized events" +python3 scripts/workflow/add_backlog.py --lane research --summary "Test whether quote freshness predicts worse downstream execution" +python3 scripts/workflow/add_backlog.py --lane bug --summary "Replay output drops pair metadata" +``` + +`scripts/workflow/open_turn.py` +- Opens a new implementation or research turn. +- Pulls selected backlog items into the active turn and removes them from `BACKLOG.md`. +- Refuses to overwrite an already-open turn unless `--force` is passed. +- Use `--commit` if you want the planning change committed automatically. + +```bash +python3 scripts/workflow/open_turn.py \ + --lane implementation \ + --title "bounded replay for durable quote history" \ + --summary "Replay recent history for the configured pair from the durable store." \ + --pick I002 \ + --pick B001 +``` + +`scripts/workflow/close_turn.py` +- Archives the current implementation or research turn into `archive/`. +- Resets the live turn file back to `idle`. +- Updates `ARCHIVE.md`. +- Use `--commit` if you want the archive change committed automatically. + +```bash +python3 scripts/workflow/close_turn.py \ + --lane implementation \ + --status passed \ + --summary "Durable history landed in cluster storage and replay works for recent windows." +``` + +Possible close statuses are: + +- `passed` +- `failed` +- `paused` +- `abandoned` + +`scripts/workflow/review_diff.sh` +- Builds a review bundle consisting of a git diff plus the adversarial review prompt. +- Intended for a separate review-only agent run. + +```bash +bash scripts/workflow/review_diff.sh HEAD~1 +bash scripts/workflow/review_diff.sh main...HEAD +``` + +### Current workflow state + +At the moment, the seeded active implementation proof is in: + +- `PROOF.md` +- `IMPLEMENTATION.md` + +That proof is focused on the first real quote -> reference-price -> decision -> +execute loop for the live configured pair, with PostgreSQL as the first durable +audit and analytics store behind the Kafka backbone. diff --git a/compose.yml b/compose.yml index 62809db..29f3259 100644 --- a/compose.yml +++ b/compose.yml @@ -1,4 +1,4 @@ -# Local/dev runtime reference. Hetzner production bootstrap now starts from Terraform + cloud-init + k3s. +# Local/dev runtime reference for the active implementation turn. services: redpanda: image: docker.redpanda.com/redpandadata/redpanda:v24.3.9 @@ -34,31 +34,84 @@ services: retries: 10 start_period: 20s + postgres: + image: postgres:16-bookworm + environment: + POSTGRES_DB: unrip + POSTGRES_USER: unrip + POSTGRES_PASSWORD: unrip + ports: + - "127.0.0.1:15432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U unrip -d unrip"] + interval: 10s + timeout: 5s + retries: 10 + near-intents-ingest: build: . command: ["node", "src/apps/near-intents-ingest.mjs"] - env_file: - - .env + env_file: [.env] depends_on: redpanda: condition: service_healthy restart: unless-stopped - dummy-reactor: + market-reference-ingest: build: . - command: ["node", "src/apps/dummy-reactor.mjs"] - env_file: - - .env + command: ["node", "src/apps/market-reference-ingest.mjs"] + env_file: [.env] depends_on: redpanda: condition: service_healthy restart: unless-stopped - dummy-executor: + liquidity-manager: build: . - command: ["node", "src/apps/dummy-executor.mjs"] - env_file: - - .env + command: ["node", "src/apps/liquidity-manager.mjs"] + env_file: [.env] + depends_on: + redpanda: + condition: service_healthy + restart: unless-stopped + volumes: + - liquidity-state:/var/lib/unrip/liquidity-state + + inventory-sync: + build: . + command: ["node", "src/apps/inventory-sync.mjs"] + env_file: [.env] + depends_on: + redpanda: + condition: service_healthy + restart: unless-stopped + + history-writer: + build: . + command: ["node", "src/apps/history-writer.mjs"] + env_file: [.env] + depends_on: + redpanda: + condition: service_healthy + postgres: + condition: service_healthy + restart: unless-stopped + + strategy-engine: + build: . + command: ["node", "src/apps/strategy-engine.mjs"] + env_file: [.env] + depends_on: + redpanda: + condition: service_healthy + restart: unless-stopped + + trade-executor: + build: . + command: ["node", "src/apps/trade-executor.mjs"] + env_file: [.env] depends_on: redpanda: condition: service_healthy @@ -66,16 +119,8 @@ services: volumes: - executor-state:/var/lib/unrip/executor-state - dummy-consumer: - build: . - command: ["node", "src/apps/dummy-consumer.mjs"] - env_file: - - .env - depends_on: - redpanda: - condition: service_healthy - restart: unless-stopped - volumes: redpanda-data: + postgres-data: executor-state: + liquidity-state: diff --git a/deploy/k8s/base/bootstrap-job.yaml b/deploy/k8s/base/bootstrap-job.yaml index 621022e..cf8d059 100644 --- a/deploy/k8s/base/bootstrap-job.yaml +++ b/deploy/k8s/base/bootstrap-job.yaml @@ -16,7 +16,7 @@ spec: - | set -eu BROKERS="redpanda.unrip.svc.cluster.local:9092" - TOPICS="raw.near_intents.quote norm.swap_demand cmd.execute_trade exec.trade_result" + TOPICS="raw.near_intents.quote norm.swap_demand ref.market_price state.intent_inventory ops.liquidity_action decision.trade_decision cmd.execute_trade exec.trade_result" RETENTION_MS="172800000" RETENTION_BYTES="268435456" diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml index 26317fe..2bcff79 100644 --- a/deploy/k8s/base/kustomization.yaml +++ b/deploy/k8s/base/kustomization.yaml @@ -3,5 +3,6 @@ kind: Kustomization resources: - namespace.yaml - redpanda.yaml + - postgres.yaml - unrip.yaml - bootstrap-job.yaml diff --git a/deploy/k8s/base/postgres.yaml b/deploy/k8s/base/postgres.yaml new file mode 100644 index 0000000..8d6c4b8 --- /dev/null +++ b/deploy/k8s/base/postgres.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-data + namespace: unrip +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: unrip +spec: + selector: + app: postgres + ports: + - name: postgres + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: unrip +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + app.kubernetes.io/part-of: unrip + spec: + containers: + - name: postgres + image: postgres:16-bookworm + ports: + - containerPort: 5432 + name: postgres + env: + - name: POSTGRES_DB + value: unrip + - name: POSTGRES_USER + value: unrip + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: unrip-secrets + key: POSTGRES_PASSWORD + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: postgres-data diff --git a/deploy/k8s/base/unrip.yaml b/deploy/k8s/base/unrip.yaml index 1c868e9..cd95af2 100644 --- a/deploy/k8s/base/unrip.yaml +++ b/deploy/k8s/base/unrip.yaml @@ -4,22 +4,69 @@ metadata: name: unrip-config namespace: unrip data: + PROJECT_NAME: unrip + PROJECT_NAMESPACE: unrip NEAR_INTENTS_WS_URL: wss://solver-relay-v2.chaindefuser.com/ws + NEAR_INTENTS_RPC_URL: https://solver-relay-v2.chaindefuser.com/rpc + NEAR_INTENTS_BRIDGE_RPC_URL: https://bridge.chaindefuser.com/rpc + NEAR_INTENTS_VERIFIER_CONTRACT: intents.near + NEAR_RPC_URL: https://rpc.fastnear.com NEAR_INTENTS_PAIR_FILTER: nep141:btc.omft.near->nep141:gnosis-0x420ca0f9b9b604ce0fd9c18ef134c705e5fa3430.omft.near + NEAR_INTENTS_ACCOUNT_ID: unrip-dev.near + TRADING_BTC_ASSET_ID: nep141:btc.omft.near + TRADING_BTC_SYMBOL: BTC + TRADING_BTC_DECIMALS: "8" + TRADING_BTC_CHAIN: btc:mainnet + TRADING_EURE_ASSET_ID: nep141:gnosis-0x420ca0f9b9b604ce0fd9c18ef134c705e5fa3430.omft.near + TRADING_EURE_SYMBOL: EURe + TRADING_EURE_DECIMALS: "18" + TRADING_EURE_CHAIN: "eth:100" NEAR_INTENTS_CONTROL_API_ENABLED: "true" NEAR_INTENTS_CONTROL_HOST: 0.0.0.0 NEAR_INTENTS_CONTROL_PORT: "8081" + MARKET_REFERENCE_CONTROL_HOST: 0.0.0.0 + MARKET_REFERENCE_CONTROL_PORT: "8082" + INVENTORY_SYNC_CONTROL_HOST: 0.0.0.0 + INVENTORY_SYNC_CONTROL_PORT: "8083" + LIQUIDITY_MANAGER_CONTROL_HOST: 0.0.0.0 + LIQUIDITY_MANAGER_CONTROL_PORT: "8084" + HISTORY_WRITER_CONTROL_HOST: 0.0.0.0 + HISTORY_WRITER_CONTROL_PORT: "8085" + STRATEGY_ENGINE_CONTROL_HOST: 0.0.0.0 + STRATEGY_ENGINE_CONTROL_PORT: "8086" + TRADE_EXECUTOR_CONTROL_HOST: 0.0.0.0 + TRADE_EXECUTOR_CONTROL_PORT: "8087" KAFKA_BROKERS: redpanda.unrip.svc.cluster.local:9092 KAFKA_CLIENT_ID: unrip KAFKA_TOPIC_RAW_NEAR_INTENTS_QUOTE: raw.near_intents.quote KAFKA_TOPIC_NORM_SWAP_DEMAND: norm.swap_demand + KAFKA_TOPIC_REF_MARKET_PRICE: ref.market_price + KAFKA_TOPIC_STATE_INTENT_INVENTORY: state.intent_inventory + KAFKA_TOPIC_OPS_LIQUIDITY_ACTION: ops.liquidity_action + KAFKA_TOPIC_DECISION_TRADE_DECISION: decision.trade_decision KAFKA_TOPIC_CMD_EXECUTE_TRADE: cmd.execute_trade KAFKA_TOPIC_EXEC_TRADE_RESULT: exec.trade_result - KAFKA_CONSUMER_GROUP_DUMMY: dummy-reactor-v1 - KAFKA_CONSUMER_GROUP_EXECUTOR: dummy-executor-v1 + KAFKA_CONSUMER_GROUP_HISTORY: history-writer-v1 + KAFKA_CONSUMER_GROUP_INVENTORY: inventory-sync-v1 + KAFKA_CONSUMER_GROUP_STRATEGY: strategy-engine-v1 + KAFKA_CONSUMER_GROUP_EXECUTOR: trade-executor-v1 EXECUTOR_STATE_DIR: /var/lib/unrip/executor-state - PROJECT_NAME: unrip - PROJECT_NAMESPACE: unrip + LIQUIDITY_STATE_DIR: /var/lib/unrip/liquidity-state + MARKET_REFERENCE_REFRESH_MS: "5000" + MARKET_REFERENCE_COINGECKO_REFRESH_MS: "15000" + MARKET_REFERENCE_MAX_AGE_MS: "30000" + MARKET_REFERENCE_KRAKEN_TICKER_URL: https://api.kraken.com/0/public/Ticker?pair=XBTEUR + MARKET_REFERENCE_COINGECKO_URL: https://api.coingecko.com/api/v3/simple/price?ids=bitcoin&vs_currencies=eur + INVENTORY_SYNC_REFRESH_MS: "15000" + LIQUIDITY_REFRESH_MS: "30000" + STRATEGY_GROSS_THRESHOLD_PCT: "2" + STRATEGY_INITIAL_ARMED: "false" + STRATEGY_MAX_NOTIONAL_EURE: "5" + STRATEGY_PRICE_MAX_AGE_MS: "30000" + STRATEGY_INVENTORY_MAX_AGE_MS: "30000" + EXECUTOR_INITIAL_ARMED: "false" + EXECUTOR_RESPONSE_TIMEOUT_MS: "10000" + LIQUIDITY_WITHDRAWALS_FROZEN: "true" --- apiVersion: v1 kind: PersistentVolumeClaim @@ -32,6 +79,17 @@ spec: requests: storage: 5Gi --- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: liquidity-state + namespace: unrip +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -67,17 +125,17 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: dummy-reactor + name: market-reference-ingest namespace: unrip spec: replicas: 1 selector: matchLabels: - app: dummy-reactor + app: market-reference-ingest template: metadata: labels: - app: dummy-reactor + app: market-reference-ingest app.kubernetes.io/part-of: unrip spec: imagePullSecrets: @@ -86,7 +144,10 @@ spec: - name: app image: ghcr.io/example/unrip:bootstrap imagePullPolicy: IfNotPresent - command: ["node", "src/apps/dummy-reactor.mjs"] + command: ["node", "src/apps/market-reference-ingest.mjs"] + ports: + - name: control-api + containerPort: 8082 envFrom: - configMapRef: name: unrip-config @@ -96,17 +157,17 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: dummy-executor + name: liquidity-manager namespace: unrip spec: replicas: 1 selector: matchLabels: - app: dummy-executor + app: liquidity-manager template: metadata: labels: - app: dummy-executor + app: liquidity-manager app.kubernetes.io/part-of: unrip spec: imagePullSecrets: @@ -115,7 +176,145 @@ spec: - name: app image: ghcr.io/example/unrip:bootstrap imagePullPolicy: IfNotPresent - command: ["node", "src/apps/dummy-executor.mjs"] + command: ["node", "src/apps/liquidity-manager.mjs"] + ports: + - name: control-api + containerPort: 8084 + envFrom: + - configMapRef: + name: unrip-config + - secretRef: + name: unrip-secrets + volumeMounts: + - name: liquidity-state + mountPath: /var/lib/unrip/liquidity-state + volumes: + - name: liquidity-state + persistentVolumeClaim: + claimName: liquidity-state +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inventory-sync + namespace: unrip +spec: + replicas: 1 + selector: + matchLabels: + app: inventory-sync + template: + metadata: + labels: + app: inventory-sync + app.kubernetes.io/part-of: unrip + spec: + imagePullSecrets: + - name: unrip-registry-creds + containers: + - name: app + image: ghcr.io/example/unrip:bootstrap + imagePullPolicy: IfNotPresent + command: ["node", "src/apps/inventory-sync.mjs"] + ports: + - name: control-api + containerPort: 8083 + envFrom: + - configMapRef: + name: unrip-config + - secretRef: + name: unrip-secrets +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: history-writer + namespace: unrip +spec: + replicas: 1 + selector: + matchLabels: + app: history-writer + template: + metadata: + labels: + app: history-writer + app.kubernetes.io/part-of: unrip + spec: + imagePullSecrets: + - name: unrip-registry-creds + containers: + - name: app + image: ghcr.io/example/unrip:bootstrap + imagePullPolicy: IfNotPresent + command: ["node", "src/apps/history-writer.mjs"] + ports: + - name: control-api + containerPort: 8085 + envFrom: + - configMapRef: + name: unrip-config + - secretRef: + name: unrip-secrets +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: strategy-engine + namespace: unrip +spec: + replicas: 1 + selector: + matchLabels: + app: strategy-engine + template: + metadata: + labels: + app: strategy-engine + app.kubernetes.io/part-of: unrip + spec: + imagePullSecrets: + - name: unrip-registry-creds + containers: + - name: app + image: ghcr.io/example/unrip:bootstrap + imagePullPolicy: IfNotPresent + command: ["node", "src/apps/strategy-engine.mjs"] + ports: + - name: control-api + containerPort: 8086 + envFrom: + - configMapRef: + name: unrip-config + - secretRef: + name: unrip-secrets +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trade-executor + namespace: unrip +spec: + replicas: 1 + selector: + matchLabels: + app: trade-executor + template: + metadata: + labels: + app: trade-executor + app.kubernetes.io/part-of: unrip + spec: + imagePullSecrets: + - name: unrip-registry-creds + containers: + - name: app + image: ghcr.io/example/unrip:bootstrap + imagePullPolicy: IfNotPresent + command: ["node", "src/apps/trade-executor.mjs"] + ports: + - name: control-api + containerPort: 8087 envFrom: - configMapRef: name: unrip-config @@ -128,32 +327,3 @@ spec: - name: executor-state persistentVolumeClaim: claimName: executor-state ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dummy-consumer - namespace: unrip -spec: - replicas: 1 - selector: - matchLabels: - app: dummy-consumer - template: - metadata: - labels: - app: dummy-consumer - app.kubernetes.io/part-of: unrip - spec: - imagePullSecrets: - - name: unrip-registry-creds - containers: - - name: app - image: ghcr.io/example/unrip:bootstrap - imagePullPolicy: IfNotPresent - command: ["node", "src/apps/dummy-consumer.mjs"] - envFrom: - - configMapRef: - name: unrip-config - - secretRef: - name: unrip-secrets diff --git a/deploy/redpanda/rpk-topics.txt b/deploy/redpanda/rpk-topics.txt index 2ba6b79..5704dfd 100644 --- a/deploy/redpanda/rpk-topics.txt +++ b/deploy/redpanda/rpk-topics.txt @@ -1,4 +1,8 @@ raw.near_intents.quote norm.swap_demand +ref.market_price +state.intent_inventory +ops.liquidity_action +decision.trade_decision cmd.execute_trade exec.trade_result diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md new file mode 100644 index 0000000..1cb27ae --- /dev/null +++ b/docs/operator-runbook.md @@ -0,0 +1,131 @@ +# Operator Runbook + +This turn implements the first funded market-maker loop for the active BTC/EURe pair on NEAR Intents. + +## Verified venue flow + +The implementation follows the official NEAR Intents market-maker path: + +1. Funding handles come from the Passive Deposit/Withdrawal Service `deposit_address` RPC for the configured treasury chains. +2. Spendable inventory comes from the Verifier internal ledger on `intents.near` via `mt_batch_balance_of`. +3. Pending deposits remain non-spendable and are tracked from `recent_deposits`. +4. Real market-maker execution is a Solver Relay `quote_response` carrying a signed `token_diff`. +5. Named NEAR accounts need the executor public key registered on `intents.near` via `add_public_key` before live submission will succeed. + +The Message Bus settles matched intents on-chain after a user accepts the quote. The executor therefore submits quote responses; it does not bridge or top up inventory on the hot path. + +## Required env and secrets + +Minimum required runtime values: + +- `NEAR_INTENTS_API_KEY` +- `NEAR_INTENTS_ACCOUNT_ID` +- `NEAR_INTENTS_SIGNER_PRIVATE_KEY` +- `POSTGRES_URL` + +Before the first live attempt on a named NEAR account, register the executor public key on `intents.near` from that named account: + +```bash +near contract call-function as-transaction \ + intents.near add_public_key json-args '{ + "public_key": "ed25519:" + }' prepaid-gas '100.0 Tgas' attached-deposit '1 yoctoNEAR' \ + sign-as network-config mainnet sign-with-keychain send +``` + +The executor stays disarmed by default even after the key is registered. + +## Local bring-up + +```bash +npm install +cp .env.example .env +# fill NEAR_INTENTS_API_KEY, NEAR_INTENTS_ACCOUNT_ID, NEAR_INTENTS_SIGNER_PRIVATE_KEY +docker compose up -d --build +``` + +Services: + +- `near-intents-ingest` +- `market-reference-ingest` +- `liquidity-manager` +- `inventory-sync` +- `history-writer` +- `strategy-engine` +- `trade-executor` + +## Control APIs + +Default local ports: + +- `8081` `near-intents-ingest` +- `8082` `market-reference-ingest` +- `8083` `inventory-sync` +- `8084` `liquidity-manager` +- `8085` `history-writer` +- `8086` `strategy-engine` +- `8087` `trade-executor` + +Common inspection: + +```bash +curl -s http://127.0.0.1:8081/healthz +curl -s http://127.0.0.1:8081/state +curl -s http://127.0.0.1:8086/state +curl -s http://127.0.0.1:8087/state +``` + +Useful controls: + +```bash +curl -s -X POST http://127.0.0.1:8082/refresh +curl -s -X POST http://127.0.0.1:8083/refresh +curl -s -X POST http://127.0.0.1:8084/refresh + +curl -s -X POST http://127.0.0.1:8086/arm +curl -s -X POST http://127.0.0.1:8086/disarm +curl -s -X PUT http://127.0.0.1:8086/limits \ + -H 'content-type: application/json' \ + -d '{"max_notional_eure":5}' + +curl -s -X POST http://127.0.0.1:8087/arm +curl -s -X POST http://127.0.0.1:8087/disarm +``` + +Track a withdrawal so it stays visible in liquidity and inventory state: + +```bash +curl -s -X POST http://127.0.0.1:8084/track-withdrawal \ + -H 'content-type: application/json' \ + -d '{"withdrawal_hash":"","asset_id":"nep141:btc.omft.near","chain":"btc:mainnet","amount":"1000"}' +``` + +## Safe arming sequence + +1. Confirm `market-reference-ingest` is publishing fresh BTC/EUR data. +2. Confirm `inventory-sync` shows credited spendable balances on `intents.near`. +3. Confirm `liquidity-manager` shows the expected deposit handle and any pending funding separately from spendable inventory. +4. Confirm `history-writer` has PostgreSQL connectivity. +5. Keep `STRATEGY_MAX_NOTIONAL_EURE=5` for the first live test. +6. Arm `strategy-engine` first. +7. Observe actionable decisions without venue errors. +8. Arm `trade-executor` only when the signer key is registered and funded inventory is already credited. + +## What to inspect after a live attempt + +- `decision.trade_decision` for the reasoning chain. +- `cmd.execute_trade` for the emitted quote response command. +- `exec.trade_result` for submission outcome. +- PostgreSQL tables: + - `swap_demand_events` + - `market_price_events` + - `intent_inventory_snapshots` + - `liquidity_actions` + - `trade_decisions` + - `execute_trade_commands` + - `trade_execution_results` + +## Still fake + +- Settlement follow-up after user quote acceptance is only visible indirectly through Solver Relay quote-status observations; the repo records the live quote-response attempt, not an end-user acceptance flow it does not control. +- The executor checks signer registration best-effort. If the verifier key-check view surface changes, the live submission itself remains the definitive signal. diff --git a/package-lock.json b/package-lock.json index 1f17217..0979284 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,497 @@ "name": "near-intents-monitor-poc", "version": "0.1.0", "dependencies": { - "kafkajs": "^2.2.4" + "kafkajs": "^2.2.4", + "near-api-js": "^7.2.0", + "pg": "^8.20.0" + } + }, + "node_modules/@noble/curves": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-2.0.1.tgz", + "integrity": "sha512-vs1Az2OOTBiP4q0pwjW5aF0xp9n4MxVrmkFBxc6EKZc6ddYx5gaZiAsZoq0uRRXWbi3AT/sBqn05eRPtn1JCPw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "2.0.1" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-2.0.1.tgz", + "integrity": "sha512-XlOlEbQcE9fmuXxrVTXCTlG2nlRXa9Rj3rr5Ue/+tX+nmkgbX720YHh0VR3hBF9xDvwnb8D2shVGOwNx+ulArw==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/base": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-2.0.0.tgz", + "integrity": "sha512-3E1kpuZginKkek01ovG8krQ0Z44E3DHPjc5S2rjJw9lZn3KSQOs8S7wqikF/AH7iRanHypj85uGyxk0XAyC37w==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@types/node": { + "version": "11.11.6", + "resolved": "https://registry.npmjs.org/@types/node/-/node-11.11.6.tgz", + "integrity": "sha512-Exw4yUWMBXM3X+8oqzJNRqZSwUAaS4+7NdvHqQuFi/d+synz++xmX3QIf+BFqneW8N31R8Ky+sikfZUXq07ggQ==", + "license": "MIT" + }, + "node_modules/available-typed-arrays": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", + "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", + "license": "MIT", + "dependencies": { + "possible-typed-array-names": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/base-x": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.11.tgz", + "integrity": "sha512-xz7wQ8xDhdyP7tQxwdteLYeFfS68tSMNCZ/Y37WJ4bhGfKPpqEIlmIyueQHqOyoPhE6xNUqjzRr8ra0eF9VRvA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/bip39": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/bip39/-/bip39-3.0.2.tgz", + "integrity": "sha512-J4E1r2N0tUylTKt07ibXvhpT2c5pyAFgvuA5q1H9uDy6dEGpjV8jmymh3MTYJDLCNbIVClSB9FbND49I6N24MQ==", + "license": "ISC", + "dependencies": { + "@types/node": "11.11.6", + "create-hash": "^1.1.0", + "pbkdf2": "^3.0.9", + "randombytes": "^2.0.1" + } + }, + "node_modules/bip39-light": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/bip39-light/-/bip39-light-1.0.7.tgz", + "integrity": "sha512-WDTmLRQUsiioBdTs9BmSEmkJza+8xfJmptsNJjxnoq3EydSa/ZBXT6rm66KoT3PJIRYMnhSKNR7S9YL1l7R40Q==", + "license": "ISC", + "dependencies": { + "create-hash": "^1.1.0", + "pbkdf2": "^3.0.9" + } + }, + "node_modules/borsh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/borsh/-/borsh-2.0.0.tgz", + "integrity": "sha512-kc9+BgR3zz9+cjbwM8ODoUB4fs3X3I5A/HtX7LZKxCLaMrEeDFoBpnhZY//DTS1VZBSs6S5v46RZRbZjRFspEg==", + "license": "Apache-2.0" + }, + "node_modules/bs58": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/bs58/-/bs58-4.0.1.tgz", + "integrity": "sha512-Ok3Wdf5vOIlBrgCvTq96gBkJw+JUEzdBgyaza5HLtPm7yTHkjRy8+JzNyHF7BHa0bNWOQIp3m5YF0nnFcOIKLw==", + "license": "MIT", + "dependencies": { + "base-x": "^3.0.2" + } + }, + "node_modules/call-bind": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", + "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.0", + "es-define-property": "^1.0.0", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/cipher-base": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.7.tgz", + "integrity": "sha512-Mz9QMT5fJe7bKI7MH31UilT5cEK5EHHRCccw/YRFsRY47AuNgaV6HY3rscp0/I4Q+tTW/5zoqpSeRRI54TkDWA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.4", + "safe-buffer": "^5.2.1", + "to-buffer": "^1.2.2" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/create-hash": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", + "license": "MIT", + "dependencies": { + "cipher-base": "^1.0.1", + "inherits": "^2.0.1", + "md5.js": "^1.3.4", + "ripemd160": "^2.0.1", + "sha.js": "^2.4.0" + } + }, + "node_modules/create-hmac": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", + "license": "MIT", + "dependencies": { + "cipher-base": "^1.0.3", + "create-hash": "^1.1.0", + "inherits": "^2.0.1", + "ripemd160": "^2.0.0", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/for-each": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", + "license": "MIT", + "dependencies": { + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "license": "MIT", + "dependencies": { + "is-property": "^1.0.2" + } + }, + "node_modules/generate-object-property": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", + "integrity": "sha512-TuOwZWgJ2VAMEGJvAyPWvpqxSANF0LDpmyHauMjFYzaACvn+QTT/AZomvPCzVBV7yDN3OmwHQ5OvHaeLKre3JQ==", + "license": "MIT", + "dependencies": { + "is-property": "^1.0.0" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hash-base": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.2.tgz", + "integrity": "sha512-Bb33KbowVTIj5s7Ked1OsqHUeCpz//tPwR+E2zJgJKo9Z5XolZ9b6bdUgjmYlwnWhoOQKoTd1TYToZGn5mAYOg==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.4", + "readable-stream": "^2.3.8", + "safe-buffer": "^5.2.1", + "to-buffer": "^1.2.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-my-ip-valid": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-my-ip-valid/-/is-my-ip-valid-1.0.1.tgz", + "integrity": "sha512-jxc8cBcOWbNK2i2aTkCZP6i7wkHF1bqKFrwEHuN5Jtg5BSaZHUZQ/JTOJwoV41YvHnOaRyWWh72T/KvfNz9DJg==", + "license": "MIT" + }, + "node_modules/is-my-json-valid": { + "version": "2.20.6", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.20.6.tgz", + "integrity": "sha512-1JQwulVNjx8UqkPE/bqDaxtH4PXCe/2VRh/y3p99heOV87HG4Id5/VfDswd+YiAfHcRTfDlWgISycnHuhZq1aw==", + "license": "MIT", + "dependencies": { + "generate-function": "^2.0.0", + "generate-object-property": "^1.1.0", + "is-my-ip-valid": "^1.0.0", + "jsonpointer": "^5.0.0", + "xtend": "^4.0.0" + } + }, + "node_modules/is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==", + "license": "MIT" + }, + "node_modules/is-typed-array": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", + "license": "MIT", + "dependencies": { + "which-typed-array": "^1.1.16" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" } }, "node_modules/kafkajs": { @@ -19,6 +509,427 @@ "engines": { "node": ">=14.0.0" } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/md5.js": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", + "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", + "license": "MIT", + "dependencies": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "node_modules/near-api-js": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/near-api-js/-/near-api-js-7.2.0.tgz", + "integrity": "sha512-byLJC+VBXe8SDCABqy5A2wXE+oHzDrn8JKoIKUnpilg1Tm9v3hejYv047hF2KUmSC5Ll7Rl5I5CrU2lk/u4ocQ==", + "license": "(MIT AND Apache-2.0)", + "dependencies": { + "@noble/curves": "2.0.1", + "@noble/hashes": "2.0.1", + "@scure/base": "2.0.0", + "borsh": "2.0.0", + "is-my-json-valid": "2.20.6", + "near-seed-phrase": "0.2.1" + }, + "engines": { + "node": ">=20.18.3", + "pnpm": ">=10.4.1" + } + }, + "node_modules/near-hd-key": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/near-hd-key/-/near-hd-key-1.2.1.tgz", + "integrity": "sha512-SIrthcL5Wc0sps+2e1xGj3zceEa68TgNZDLuCx0daxmfTP7sFTB3/mtE2pYhlFsCxWoMn+JfID5E1NlzvvbRJg==", + "license": "MIT", + "dependencies": { + "bip39": "3.0.2", + "create-hmac": "1.1.7", + "tweetnacl": "1.0.3" + } + }, + "node_modules/near-seed-phrase": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/near-seed-phrase/-/near-seed-phrase-0.2.1.tgz", + "integrity": "sha512-feMuums+kVL3LSuPcP4ld07xHCb2mu6z48SGfP3W+8tl1Qm5xIcjiQzY2IDPBvFgajRDxWSb8GzsRHoInazByw==", + "license": "MIT", + "dependencies": { + "bip39-light": "^1.0.7", + "bs58": "^4.0.1", + "near-hd-key": "^1.2.1", + "tweetnacl": "^1.0.2" + } + }, + "node_modules/pbkdf2": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.5.tgz", + "integrity": "sha512-Q3CG/cYvCO1ye4QKkuH7EXxs3VC/rI1/trd+qX2+PolbaKG0H+bgcZzrTt96mMyRtejk+JMCiLUn3y29W8qmFQ==", + "license": "MIT", + "dependencies": { + "create-hash": "^1.2.0", + "create-hmac": "^1.1.7", + "ripemd160": "^2.0.3", + "safe-buffer": "^5.2.1", + "sha.js": "^2.4.12", + "to-buffer": "^1.2.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.12.0", + "pg-pool": "^3.13.0", + "pg-protocol": "^1.13.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.12.0.tgz", + "integrity": "sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.13.0.tgz", + "integrity": "sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.13.0.tgz", + "integrity": "sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/possible-typed-array-names": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", + "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readable-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/ripemd160": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.3.tgz", + "integrity": "sha512-5Di9UC0+8h1L6ZD2d7awM7E/T4uA1fJRlx6zk/NvdCCVEoAnFqvHmCuNeIKoCeIixBX/q8uM+6ycDvF8woqosA==", + "license": "MIT", + "dependencies": { + "hash-base": "^3.1.2", + "inherits": "^2.0.4" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/sha.js": { + "version": "2.4.12", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.12.tgz", + "integrity": "sha512-8LzC5+bvI45BjpfXU8V5fdU2mfeKiQe1D1gIMn7XUlF3OTUrpdJpPPH4EMAnF0DsHHdSZqCdSss5qCmJKuiO3w==", + "license": "(MIT AND BSD-3-Clause)", + "dependencies": { + "inherits": "^2.0.4", + "safe-buffer": "^5.2.1", + "to-buffer": "^1.2.0" + }, + "bin": { + "sha.js": "bin.js" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/to-buffer": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.2.2.tgz", + "integrity": "sha512-db0E3UJjcFhpDhAF4tLo03oli3pwl3dbnzXOUIlRKrp+ldk/VUxzpWYZENsw2SZiuBjHAk7DfB0VU7NKdpb6sw==", + "license": "MIT", + "dependencies": { + "isarray": "^2.0.5", + "safe-buffer": "^5.2.1", + "typed-array-buffer": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/to-buffer/node_modules/isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "license": "MIT" + }, + "node_modules/tweetnacl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", + "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==", + "license": "Unlicense" + }, + "node_modules/typed-array-buffer": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", + "integrity": "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.3", + "es-errors": "^1.3.0", + "is-typed-array": "^1.1.14" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/which-typed-array": { + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.20.tgz", + "integrity": "sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg==", + "license": "MIT", + "dependencies": { + "available-typed-arrays": "^1.0.7", + "call-bind": "^1.0.8", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } } } } diff --git a/package.json b/package.json index 73915ca..47b825c 100644 --- a/package.json +++ b/package.json @@ -5,12 +5,18 @@ "type": "module", "scripts": { "near-intents:ingest": "node src/apps/near-intents-ingest.mjs", - "dummy-reactor": "node src/apps/dummy-reactor.mjs", - "dummy-executor": "node src/apps/dummy-executor.mjs", - "dummy-consumer": "node src/apps/dummy-consumer.mjs", - "start": "node index.mjs" + "market-reference:ingest": "node src/apps/market-reference-ingest.mjs", + "inventory:sync": "node src/apps/inventory-sync.mjs", + "liquidity:manager": "node src/apps/liquidity-manager.mjs", + "history:writer": "node src/apps/history-writer.mjs", + "strategy:engine": "node src/apps/strategy-engine.mjs", + "trade:executor": "node src/apps/trade-executor.mjs", + "start": "node index.mjs", + "test": "node --test" }, "dependencies": { - "kafkajs": "^2.2.4" + "kafkajs": "^2.2.4", + "near-api-js": "^7.2.0", + "pg": "^8.20.0" } } diff --git a/scripts/deploy/bootstrap.sh b/scripts/deploy/bootstrap.sh index 57e4a41..ec2915c 100755 --- a/scripts/deploy/bootstrap.sh +++ b/scripts/deploy/bootstrap.sh @@ -10,7 +10,7 @@ FORGEJO_REMOTE_NAME="${FORGEJO_REMOTE_NAME:-forgejo}" PROJECT_NAME="${PROJECT_NAME:-unrip}" PROJECT_NAMESPACE="${PROJECT_NAMESPACE:-$PROJECT_NAME}" -PROJECT_DEPLOYMENTS="${PROJECT_DEPLOYMENTS:-near-intents-ingest,dummy-reactor,dummy-executor,dummy-consumer}" +PROJECT_DEPLOYMENTS="${PROJECT_DEPLOYMENTS:-near-intents-ingest,market-reference-ingest,liquidity-manager,inventory-sync,history-writer,strategy-engine,trade-executor}" PROJECT_REGISTRY_SECRET_NAME="${PROJECT_REGISTRY_SECRET_NAME:-${PROJECT_NAME}-registry-creds}" APP_SECRET_NAME="${APP_SECRET_NAME:-${PROJECT_NAME}-secrets}" SYNC_FORGEJO_REMOTE="${SYNC_FORGEJO_REMOTE:-1}" @@ -132,13 +132,53 @@ fi : "${REGISTRY_USERNAME:?set REGISTRY_USERNAME or bootstrap the shared registry first}" : "${REGISTRY_PASSWORD:?set REGISTRY_PASSWORD}" : "${NEAR_INTENTS_API_KEY:?set NEAR_INTENTS_API_KEY}" +: "${POSTGRES_PASSWORD:=}" +: "${POSTGRES_URL:=}" +: "${NEAR_INTENTS_SIGNER_PRIVATE_KEY:=}" + +secret_value() { + local key="$1" + kubectl -n "$PROJECT_NAMESPACE" get secret "$APP_SECRET_NAME" -o "jsonpath={.data.${key}}" 2>/dev/null | base64 -d 2>/dev/null || true +} + +if [[ -z "$POSTGRES_PASSWORD" ]]; then + POSTGRES_PASSWORD="$(secret_value POSTGRES_PASSWORD)" +fi + +if [[ -z "$POSTGRES_URL" ]]; then + POSTGRES_URL="$(secret_value POSTGRES_URL)" +fi + +if [[ -z "$NEAR_INTENTS_SIGNER_PRIVATE_KEY" ]]; then + NEAR_INTENTS_SIGNER_PRIVATE_KEY="$(secret_value NEAR_INTENTS_SIGNER_PRIVATE_KEY)" +fi + +if [[ -z "$POSTGRES_PASSWORD" ]]; then + POSTGRES_PASSWORD="$(python3 - <<'PY' +import secrets +print(secrets.token_urlsafe(24)) +PY +)" +fi + +if [[ -z "$POSTGRES_URL" ]]; then + POSTGRES_URL="postgresql://unrip:${POSTGRES_PASSWORD}@postgres:5432/unrip" +fi echo "bootstrapping namespace $PROJECT_NAMESPACE" kubectl apply -f "$ROOT_DIR/deploy/k8s/base/namespace.yaml" echo "upserting runtime secret $APP_SECRET_NAME" +secret_args=( + --from-literal=NEAR_INTENTS_API_KEY="$NEAR_INTENTS_API_KEY" + --from-literal=POSTGRES_PASSWORD="$POSTGRES_PASSWORD" + --from-literal=POSTGRES_URL="$POSTGRES_URL" +) +if [[ -n "$NEAR_INTENTS_SIGNER_PRIVATE_KEY" ]]; then + secret_args+=(--from-literal=NEAR_INTENTS_SIGNER_PRIVATE_KEY="$NEAR_INTENTS_SIGNER_PRIVATE_KEY") +fi kubectl -n "$PROJECT_NAMESPACE" create secret generic "$APP_SECRET_NAME" \ - --from-literal=NEAR_INTENTS_API_KEY="$NEAR_INTENTS_API_KEY" \ + "${secret_args[@]}" \ --dry-run=client -o yaml | kubectl apply -f - echo "upserting registry pull/push secret $PROJECT_REGISTRY_SECRET_NAME" diff --git a/src/apps/dummy-consumer.mjs b/src/apps/dummy-consumer.mjs deleted file mode 100644 index cce534b..0000000 --- a/src/apps/dummy-consumer.mjs +++ /dev/null @@ -1,61 +0,0 @@ -import process from 'node:process'; - -import { createConsumer } from '../bus/kafka/consumer.mjs'; -import { createLogger, serializeError } from '../core/log.mjs'; -import { parseEventMessage } from '../core/event-envelope.mjs'; -import { assertTradeResult } from '../core/schemas.mjs'; -import { loadConfig } from '../lib/config.mjs'; - -const config = loadConfig(); -const logger = createLogger({ - service: 'dummy-consumer', - component: 'consumer', - namespace: config.projectNamespace, -}); - -const consumer = await createConsumer({ - groupId: `${config.kafkaConsumerGroupExecutor}-results-view`, - brokers: config.kafkaBrokers, - clientId: config.kafkaClientId, - logger, -}); -await consumer.subscribe({ topic: config.kafkaTopicExecTradeResult, fromBeginning: false }); - -process.on('SIGINT', async () => { - await consumer.disconnect(); - process.exit(0); -}); -process.on('SIGTERM', async () => { - await consumer.disconnect(); - process.exit(0); -}); - -await consumer.run({ - eachMessage: async ({ message }) => { - if (!message.value) return; - - let event; - try { - event = parseEventMessage(message.value.toString()); - } catch { - logger.warn('invalid_json_message', { - topic: config.kafkaTopicExecTradeResult, - }); - return; - } - - try { - assertTradeResult(event); - } catch (error) { - logger.error('message_processing_failed', { - venue: event.venue || 'near-intents', - topic: config.kafkaTopicExecTradeResult, - details: { - error: serializeError(error), - command_id: event.payload?.command_id, - quote_id: event.payload?.quote_id, - }, - }); - } - }, -}); diff --git a/src/apps/dummy-executor.mjs b/src/apps/dummy-executor.mjs deleted file mode 100644 index ffe0969..0000000 --- a/src/apps/dummy-executor.mjs +++ /dev/null @@ -1,135 +0,0 @@ -import process from 'node:process'; - -import { createConsumer } from '../bus/kafka/consumer.mjs'; -import { createProducer } from '../bus/kafka/producer.mjs'; -import { buildEventEnvelope, parseEventMessage } from '../core/event-envelope.mjs'; -import { createExecutorStateStore } from '../core/executor-state-store.mjs'; -import { createLogger, serializeError } from '../core/log.mjs'; -import { assertExecuteTradeCommand, assertTradeResult } from '../core/schemas.mjs'; -import { loadConfig } from '../lib/config.mjs'; - -const config = loadConfig(); -const logger = createLogger({ - service: 'dummy-executor', - component: 'executor', - namespace: config.projectNamespace, -}); - -const consumer = await createConsumer({ - groupId: config.kafkaConsumerGroupExecutor, - brokers: config.kafkaBrokers, - clientId: config.kafkaClientId, - logger, -}); -const producer = await createProducer({ - brokers: config.kafkaBrokers, - clientId: config.kafkaClientId, - logger, -}); - -const stateStore = createExecutorStateStore({ stateDir: config.executorStateDir }); - -await consumer.subscribe({ topic: config.kafkaTopicCmdExecuteTrade, fromBeginning: false }); - -async function shutdown() { - await consumer.disconnect(); - await producer.disconnect(); - process.exit(0); -} -process.on('SIGINT', shutdown); -process.on('SIGTERM', shutdown); - -await consumer.run({ - eachMessage: async ({ message }) => { - if (!message.value) return; - - let event; - try { - event = parseEventMessage(message.value.toString()); - } catch { - logger.warn('invalid_json_message', { - topic: config.kafkaTopicCmdExecuteTrade, - }); - return; - } - - try { - assertExecuteTradeCommand(event); - - const payload = event.payload; - const commandId = payload.command_id; - const pair = `${payload.asset_in}->${payload.asset_out}`; - const existing = stateStore.get(commandId); - if (existing?.status === 'completed') { - logger.warn('duplicate_command_skipped', { - venue: event.venue || 'near-intents', - topic: config.kafkaTopicCmdExecuteTrade, - pair, - details: { - command_id: commandId, - quote_id: payload.quote_id, - }, - }); - return; - } - - stateStore.markProcessing(commandId, { - idempotency_key: payload.idempotency_key, - execution_key: payload.execution_key, - quote_id: payload.quote_id, - }); - - const recoveredInflight = existing?.status === 'processing'; - const result = buildEventEnvelope({ - source: 'dummy-executor', - venue: event.venue || 'near-intents', - eventType: 'trade_result', - eventId: `exec-${commandId}`, - observedAt: event.observed_at, - payload: { - command_id: commandId, - idempotency_key: payload.idempotency_key, - execution_key: payload.execution_key, - quote_id: payload.quote_id, - status: 'simulated_sent', - result_code: recoveredInflight ? 'recovered_inflight' : 'sent', - note: 'dummy executor placeholder result', - }, - }); - assertTradeResult(result); - - await producer.sendJson(config.kafkaTopicExecTradeResult, result, { key: payload.execution_key }); - stateStore.markCompleted(commandId, { - idempotency_key: payload.idempotency_key, - execution_key: payload.execution_key, - quote_id: payload.quote_id, - result_event_id: result.event_id, - }); - - if (recoveredInflight) { - logger.warn('inflight_command_recovered', { - venue: event.venue || 'near-intents', - topic: config.kafkaTopicExecTradeResult, - pair, - details: { - command_id: commandId, - quote_id: payload.quote_id, - }, - }); - } - } catch (error) { - logger.error('message_processing_failed', { - venue: event.venue || 'near-intents', - topic: config.kafkaTopicCmdExecuteTrade, - pair: event.payload?.asset_in && event.payload?.asset_out - ? `${event.payload.asset_in}->${event.payload.asset_out}` - : undefined, - details: { - error: serializeError(error), - command_id: event.payload?.command_id, - quote_id: event.payload?.quote_id, - }, - }); - } - }, -}); diff --git a/src/apps/dummy-reactor.mjs b/src/apps/dummy-reactor.mjs deleted file mode 100644 index 4854d29..0000000 --- a/src/apps/dummy-reactor.mjs +++ /dev/null @@ -1,96 +0,0 @@ -import process from 'node:process'; - -import { createConsumer } from '../bus/kafka/consumer.mjs'; -import { createProducer } from '../bus/kafka/producer.mjs'; -import { createLogger, serializeError } from '../core/log.mjs'; -import { loadConfig } from '../lib/config.mjs'; -import { buildEventEnvelope, parseEventMessage } from '../core/event-envelope.mjs'; -import { assertExecuteTradeCommand, assertNormalizedSwapDemand } from '../core/schemas.mjs'; - -const config = loadConfig(); -const logger = createLogger({ - service: 'dummy-reactor', - component: 'reactor', - namespace: config.projectNamespace, -}); - -const consumer = await createConsumer({ - groupId: config.kafkaConsumerGroupDummy, - brokers: config.kafkaBrokers, - clientId: config.kafkaClientId, - logger, -}); -const producer = await createProducer({ - brokers: config.kafkaBrokers, - clientId: config.kafkaClientId, - logger, -}); - -await consumer.subscribe({ topic: config.kafkaTopicNormSwapDemand, fromBeginning: false }); - -async function shutdown() { - await consumer.disconnect(); - await producer.disconnect(); - process.exit(0); -} -process.on('SIGINT', shutdown); -process.on('SIGTERM', shutdown); - -await consumer.run({ - eachMessage: async ({ message }) => { - if (!message.value) return; - - let event; - try { - event = parseEventMessage(message.value.toString()); - } catch { - logger.warn('invalid_json_message', { - topic: config.kafkaTopicNormSwapDemand, - }); - return; - } - - try { - assertNormalizedSwapDemand(event); - - const payload = event.payload; - const pair = `${payload.asset_in}->${payload.asset_out}`; - const quoteId = payload.quote_id; - const commandId = `cmd-${quoteId}`; - const command = buildEventEnvelope({ - source: 'dummy-reactor', - venue: event.venue || 'near-intents', - eventType: 'execute_trade', - eventId: commandId, - observedAt: event.observed_at, - payload: { - command_id: commandId, - idempotency_key: `${event.venue || 'near-intents'}:${quoteId}`, - execution_key: `${event.venue || 'near-intents'}:${payload.asset_in}->${payload.asset_out}`, - quote_id: quoteId, - asset_in: payload.asset_in, - asset_out: payload.asset_out, - amount_in: payload.amount_in, - amount_out: payload.amount_out, - reason: 'dummy reactor placeholder decision', - }, - }); - assertExecuteTradeCommand(command); - - await producer.sendJson(config.kafkaTopicCmdExecuteTrade, command, { key: command.payload.execution_key }); - return; - } catch (error) { - logger.error('message_processing_failed', { - venue: event.venue || 'near-intents', - topic: config.kafkaTopicNormSwapDemand, - pair: event.payload?.asset_in && event.payload?.asset_out - ? `${event.payload.asset_in}->${event.payload.asset_out}` - : undefined, - details: { - error: serializeError(error), - quote_id: event.payload?.quote_id, - }, - }); - } - }, -}); diff --git a/src/apps/history-writer.mjs b/src/apps/history-writer.mjs new file mode 100644 index 0000000..22a8d45 --- /dev/null +++ b/src/apps/history-writer.mjs @@ -0,0 +1,147 @@ +import process from 'node:process'; + +import { createConsumer } from '../bus/kafka/consumer.mjs'; +import { startControlApi } from '../core/control-api.mjs'; +import { routeHistoryRecord } from '../core/history-records.mjs'; +import { createLogger, serializeError } from '../core/log.mjs'; +import { parseEventMessage } from '../core/event-envelope.mjs'; +import { loadConfig } from '../lib/config.mjs'; +import { createPostgresPool, ensureHistorySchema, insertHistoryEvent } from '../lib/postgres.mjs'; + +const config = loadConfig(); +const logger = createLogger({ + service: 'history-writer', + component: 'history', + namespace: config.projectNamespace, +}); + +const pool = createPostgresPool({ + connectionString: config.postgresUrl, +}); +await ensureHistorySchema(pool); + +const consumer = await createConsumer({ + groupId: config.kafkaConsumerGroupHistory, + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, + logger, +}); + +const topics = [ + config.kafkaTopicRawNearIntentsQuote, + config.kafkaTopicNormSwapDemand, + config.kafkaTopicRefMarketPrice, + config.kafkaTopicStateIntentInventory, + config.kafkaTopicOpsLiquidityAction, + config.kafkaTopicDecisionTradeDecision, + config.kafkaTopicCmdExecuteTrade, + config.kafkaTopicExecTradeResult, +]; + +for (const topic of topics) { + await consumer.subscribe({ topic, fromBeginning: true }); +} + +const state = { + paused: false, + draining: false, + last_write_at: null, + last_error: null, + error_count: 0, + offsets: {}, +}; + +await consumer.run({ + eachMessage: async ({ topic, partition, message }) => { + if (!message.value || state.paused) return; + + try { + const event = parseEventMessage(message.value.toString()); + const routed = routeHistoryRecord({ topic, event }); + await insertHistoryEvent(pool, { + table: routed.table, + topic, + event, + record: routed.record, + }); + + state.last_write_at = new Date().toISOString(); + state.last_error = null; + state.offsets[topic] = { + partition, + offset: message.offset, + }; + } catch (error) { + state.last_error = serializeError(error); + state.error_count += 1; + logger.error('history_write_failed', { + topic, + details: { + error: serializeError(error), + }, + }); + } finally { + if (state.draining) { + setTimeout(() => shutdown(), 0); + } + } + }, +}); + +const controlApi = startControlApi({ + host: config.historyWriterControlHost, + port: config.historyWriterControlPort, + logger: logger.child({ component: 'control-api' }), + service: 'history-writer', + namespace: config.projectNamespace, + stateProvider: { + async getState() { + const connectivity = await pool.query('SELECT 1').then(() => true).catch(() => false); + return { + ...state, + database_connectivity: connectivity, + }; + }, + }, + routes: [ + { + method: 'POST', + path: '/pause', + handler: () => { + state.paused = true; + consumer.pause(topics.map((topic) => ({ topic }))); + return { ok: true, paused: true }; + }, + }, + { + method: 'POST', + path: '/resume', + handler: () => { + state.paused = false; + consumer.resume(topics.map((topic) => ({ topic }))); + return { ok: true, paused: false }; + }, + }, + { + method: 'POST', + path: '/drain', + handler: () => { + state.draining = true; + state.paused = true; + consumer.pause(topics.map((topic) => ({ topic }))); + setTimeout(() => shutdown(), 0); + return { ok: true, draining: true }; + }, + }, + ], +}); + +async function shutdown() { + await controlApi.close().catch(() => {}); + await consumer.disconnect(); + await pool.end(); + process.exit(0); +} + +process.on('SIGINT', shutdown); +process.on('SIGTERM', shutdown); diff --git a/src/apps/inventory-sync.mjs b/src/apps/inventory-sync.mjs new file mode 100644 index 0000000..6d067a9 --- /dev/null +++ b/src/apps/inventory-sync.mjs @@ -0,0 +1,206 @@ +import process from 'node:process'; + +import { createConsumer } from '../bus/kafka/consumer.mjs'; +import { createProducer } from '../bus/kafka/producer.mjs'; +import { startControlApi } from '../core/control-api.mjs'; +import { buildEventEnvelope, parseEventMessage } from '../core/event-envelope.mjs'; +import { buildInventorySnapshot } from '../core/inventory.mjs'; +import { createLogger, serializeError } from '../core/log.mjs'; +import { assertInventorySnapshotEvent, assertLiquidityActionEvent } from '../core/schemas.mjs'; +import { loadConfig } from '../lib/config.mjs'; +import { createNearBridgeClient } from '../venues/near-intents/bridge-client.mjs'; +import { createVerifierClient } from '../venues/near-intents/verifier-client.mjs'; + +const config = loadConfig(); +const logger = createLogger({ + service: 'inventory-sync', + component: 'inventory', + namespace: config.projectNamespace, + venue: 'near-intents', +}); + +if (!config.nearIntentsAccountId) { + logger.error('missing_account_id', { + details: { + variable: 'NEAR_INTENTS_ACCOUNT_ID', + }, + }); + process.exit(1); +} + +const bridgeClient = createNearBridgeClient({ rpcUrl: config.nearBridgeRpcUrl }); +const verifierClient = createVerifierClient({ + nearRpcUrl: config.nearRpcUrl, + verifierContract: config.nearVerifierContract, +}); +const producer = await createProducer({ + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, + logger, +}); +const consumer = await createConsumer({ + groupId: config.kafkaConsumerGroupInventory, + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, + logger, +}); + +const state = { + paused: false, + tracked_withdrawals: {}, + last_snapshot: null, + last_sync_at: null, + last_error: null, + publish_count: 0, +}; + +await consumer.subscribe({ topic: config.kafkaTopicOpsLiquidityAction, fromBeginning: true }); +await consumer.run({ + eachMessage: async ({ message }) => { + if (!message.value) return; + try { + const event = parseEventMessage(message.value.toString()); + assertLiquidityActionEvent(event); + if (event.payload.action_type === 'withdrawal_tracked' + || event.payload.action_type === 'withdrawal_status_changed') { + const details = event.payload.details || {}; + if (details.withdrawal_hash) { + state.tracked_withdrawals[details.withdrawal_hash] = { + withdrawal_hash: details.withdrawal_hash, + asset_id: details.asset_id, + chain: details.chain, + amount: String(details.amount || '0'), + status: details.status || event.payload.status, + address: details.address || null, + }; + } + } + } catch (error) { + logger.error('liquidity_action_consume_failed', { + topic: config.kafkaTopicOpsLiquidityAction, + details: { + error: serializeError(error), + }, + }); + } + }, +}); + +async function refresh() { + if (state.paused) return; + + try { + const balances = await verifierClient.mtBatchBalanceOf({ + accountId: config.nearIntentsAccountId, + tokenIds: config.activeAssetIds, + }); + const recentDeposits = []; + for (const chain of [config.tradingBtc.chain, config.tradingEure.chain]) { + const response = await bridgeClient.recentDeposits({ + accountId: config.nearIntentsAccountId, + chain, + }); + for (const deposit of response?.deposits || []) { + recentDeposits.push({ + tx_hash: deposit.tx_hash || null, + chain, + asset_id: chain === config.tradingBtc.chain ? config.tradingBtc.assetId : config.tradingEure.assetId, + amount: String(deposit.amount || '0'), + address: deposit.address, + status: deposit.status, + decimals: deposit.decimals, + }); + } + } + + const snapshot = buildInventorySnapshot({ + accountId: config.nearIntentsAccountId, + balances, + recentDeposits, + trackedWithdrawals: Object.values(state.tracked_withdrawals), + assetRegistry: config.assetRegistry, + observedAt: new Date().toISOString(), + }); + state.last_snapshot = snapshot; + state.last_sync_at = snapshot.synced_at; + state.last_error = null; + + const event = buildEventEnvelope({ + source: 'inventory-sync', + venue: 'near-intents', + eventType: 'intent_inventory', + observedAt: snapshot.synced_at, + payload: snapshot, + }); + assertInventorySnapshotEvent(event); + await producer.sendJson(config.kafkaTopicStateIntentInventory, event, { key: snapshot.inventory_id }); + state.publish_count += 1; + } catch (error) { + state.last_error = serializeError(error); + logger.error('inventory_refresh_failed', { + topic: config.kafkaTopicStateIntentInventory, + pair: config.activePair, + details: { + error: serializeError(error), + }, + }); + } +} + +const timer = setInterval(refresh, config.inventorySyncRefreshMs); +timer.unref?.(); +await refresh(); + +const controlApi = startControlApi({ + host: config.inventorySyncControlHost, + port: config.inventorySyncControlPort, + logger: logger.child({ component: 'control-api' }), + service: 'inventory-sync', + namespace: config.projectNamespace, + stateProvider: { + getState() { + return { + account_id: config.nearIntentsAccountId, + ...state, + }; + }, + }, + routes: [ + { + method: 'POST', + path: '/refresh', + handler: async () => { + await refresh(); + return { ok: true, ...state }; + }, + }, + { + method: 'POST', + path: '/pause', + handler: () => { + state.paused = true; + return { ok: true, paused: true }; + }, + }, + { + method: 'POST', + path: '/resume', + handler: async () => { + state.paused = false; + await refresh(); + return { ok: true, paused: false }; + }, + }, + ], +}); + +async function shutdown() { + clearInterval(timer); + await controlApi.close().catch(() => {}); + await consumer.disconnect(); + await producer.disconnect(); + process.exit(0); +} + +process.on('SIGINT', shutdown); +process.on('SIGTERM', shutdown); diff --git a/src/apps/liquidity-manager.mjs b/src/apps/liquidity-manager.mjs new file mode 100644 index 0000000..4a0f84d --- /dev/null +++ b/src/apps/liquidity-manager.mjs @@ -0,0 +1,327 @@ +import process from 'node:process'; + +import { createProducer } from '../bus/kafka/producer.mjs'; +import { startControlApi } from '../core/control-api.mjs'; +import { buildEventEnvelope } from '../core/event-envelope.mjs'; +import { createJsonStateStore } from '../core/json-state-store.mjs'; +import { createLogger, serializeError } from '../core/log.mjs'; +import { assertLiquidityActionEvent } from '../core/schemas.mjs'; +import { loadConfig } from '../lib/config.mjs'; +import { createNearBridgeClient } from '../venues/near-intents/bridge-client.mjs'; + +const config = loadConfig(); +const logger = createLogger({ + service: 'liquidity-manager', + component: 'funding', + namespace: config.projectNamespace, + venue: 'near-intents', +}); + +if (!config.nearIntentsAccountId) { + logger.error('missing_account_id', { + details: { + variable: 'NEAR_INTENTS_ACCOUNT_ID', + }, + }); + process.exit(1); +} + +const producer = await createProducer({ + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, + logger, +}); +const bridgeClient = createNearBridgeClient({ rpcUrl: config.nearBridgeRpcUrl }); +const store = createJsonStateStore({ + stateDir: config.liquidityStateDir, + fileName: 'liquidity.json', + initialState: { + paused: false, + withdrawals_frozen: config.withdrawalsFrozen, + deposit_addresses: {}, + deposits: {}, + tracked_withdrawals: {}, + supported_tokens: {}, + last_refresh_at: null, + last_error: null, + publish_count: 0, + }, +}); + +const chains = [config.tradingBtc.chain, config.tradingEure.chain]; +const assetsByChain = new Map([ + [config.tradingBtc.chain, config.tradingBtc.assetId], + [config.tradingEure.chain, config.tradingEure.assetId], +]); + +async function refresh() { + const state = store.getState(); + if (state.paused) return; + + try { + const supported = await bridgeClient.supportedTokens({ chains }); + state.supported_tokens = mapSupportedTokens(supported?.tokens || []); + + for (const chain of chains) { + await refreshChain(chain, state); + } + + for (const tracked of Object.values(state.tracked_withdrawals)) { + await refreshWithdrawal(tracked, state); + } + + state.last_refresh_at = new Date().toISOString(); + state.last_error = null; + store.setState(state); + } catch (error) { + state.last_error = serializeError(error); + store.setState(state); + logger.error('liquidity_refresh_failed', { + topic: config.kafkaTopicOpsLiquidityAction, + details: { + error: serializeError(error), + }, + }); + } +} + +async function refreshChain(chain, state) { + const depositAddress = await bridgeClient.depositAddress({ + accountId: config.nearIntentsAccountId, + chain, + }); + const previousAddress = state.deposit_addresses[chain]?.address || null; + state.deposit_addresses[chain] = { + ...(state.deposit_addresses[chain] || {}), + ...depositAddress, + refreshed_at: new Date().toISOString(), + }; + + if (previousAddress !== depositAddress.address) { + await publishAction({ + action_type: 'deposit_address_refreshed', + status: 'READY', + chain, + asset_id: assetsByChain.get(chain), + details: depositAddress, + }, state); + } + + const deposits = await bridgeClient.recentDeposits({ + accountId: config.nearIntentsAccountId, + chain, + }); + for (const deposit of deposits?.deposits || []) { + const key = `${chain}:${deposit.tx_hash || deposit.address}:${deposit.defuse_asset_identifier}`; + const assetId = mapDepositAssetId(deposit.defuse_asset_identifier, chain); + const normalized = { + tx_hash: deposit.tx_hash || null, + chain, + asset_id: assetId, + amount: String(deposit.amount || '0'), + account_id: deposit.account_id, + address: deposit.address, + status: deposit.status, + decimals: deposit.decimals, + }; + const previous = state.deposits[key]; + state.deposits[key] = normalized; + if (!previous || previous.status !== normalized.status) { + await publishAction({ + action_type: 'deposit_status_observed', + status: normalized.status, + chain, + asset_id: assetId, + details: normalized, + }, state); + } + } +} + +async function refreshWithdrawal(tracked, state) { + const status = await bridgeClient.withdrawalStatus({ + withdrawalHash: tracked.withdrawal_hash, + }); + const next = { + ...tracked, + status: status.status, + transfer_tx_hash: status.data?.transfer_tx_hash || null, + last_checked_at: new Date().toISOString(), + }; + state.tracked_withdrawals[tracked.withdrawal_hash] = next; + + if (tracked.status !== next.status) { + await publishAction({ + action_type: 'withdrawal_status_changed', + status: next.status, + chain: next.chain, + asset_id: next.asset_id, + details: next, + }, state); + } +} + +async function publishAction(payload, state) { + const event = buildEventEnvelope({ + source: 'liquidity-manager', + venue: 'near-intents', + eventType: 'liquidity_action', + payload: { + liquidity_action_id: `${payload.action_type}-${Date.now()}-${Math.random().toString(16).slice(2, 8)}`, + account_id: config.nearIntentsAccountId, + ...payload, + }, + }); + assertLiquidityActionEvent(event); + await producer.sendJson(config.kafkaTopicOpsLiquidityAction, event, { key: event.payload.liquidity_action_id }); + state.publish_count += 1; +} + +const timer = setInterval(refresh, config.liquidityRefreshMs); +timer.unref?.(); +await refresh(); + +const controlApi = startControlApi({ + host: config.liquidityManagerControlHost, + port: config.liquidityManagerControlPort, + logger: logger.child({ component: 'control-api' }), + service: 'liquidity-manager', + namespace: config.projectNamespace, + stateProvider: { + getState() { + return { + account_id: config.nearIntentsAccountId, + ...store.getState(), + }; + }, + }, + routes: [ + { + method: 'POST', + path: '/refresh', + handler: async () => { + await refresh(); + return { + ok: true, + ...store.getState(), + }; + }, + }, + { + method: 'POST', + path: '/pause', + handler: () => { + const state = store.getState(); + state.paused = true; + store.setState(state); + return { ok: true, paused: true }; + }, + }, + { + method: 'POST', + path: '/resume', + handler: async () => { + const state = store.getState(); + state.paused = false; + store.setState(state); + await refresh(); + return { ok: true, paused: false }; + }, + }, + { + method: 'POST', + path: '/freeze-withdrawals', + handler: async ({ body }) => { + const state = store.getState(); + state.withdrawals_frozen = body.frozen !== false; + store.setState(state); + await publishAction({ + action_type: 'withdrawals_frozen', + status: state.withdrawals_frozen ? 'FROZEN' : 'UNFROZEN', + chain: null, + asset_id: null, + details: { + frozen: state.withdrawals_frozen, + }, + }, state); + store.setState(state); + return { ok: true, withdrawals_frozen: state.withdrawals_frozen }; + }, + }, + { + method: 'POST', + path: '/track-withdrawal', + handler: async ({ body }) => { + if (!body.withdrawal_hash || !body.asset_id || !body.chain) { + return { + statusCode: 400, + payload: { error: 'withdrawal_hash, asset_id, and chain are required' }, + }; + } + + const state = store.getState(); + state.tracked_withdrawals[body.withdrawal_hash] = { + withdrawal_hash: body.withdrawal_hash, + asset_id: body.asset_id, + chain: body.chain, + amount: String(body.amount || '0'), + status: 'TRACKED', + address: body.address || null, + noted_at: new Date().toISOString(), + }; + store.setState(state); + await publishAction({ + action_type: 'withdrawal_tracked', + status: 'TRACKED', + chain: body.chain, + asset_id: body.asset_id, + details: state.tracked_withdrawals[body.withdrawal_hash], + }, state); + store.setState(state); + return { ok: true, tracked: state.tracked_withdrawals[body.withdrawal_hash] }; + }, + }, + { + method: 'POST', + path: '/notify-deposit', + handler: async ({ body }) => { + if (!body.deposit_address || !body.tx_hash) { + return { + statusCode: 400, + payload: { error: 'deposit_address and tx_hash are required' }, + }; + } + await bridgeClient.notifyDeposit({ + depositAddress: body.deposit_address, + txHash: body.tx_hash, + }); + return { ok: true }; + }, + }, + ], +}); + +async function shutdown() { + clearInterval(timer); + await controlApi.close().catch(() => {}); + await producer.disconnect(); + process.exit(0); +} + +process.on('SIGINT', shutdown); +process.on('SIGTERM', shutdown); + +function mapSupportedTokens(tokens) { + return Object.fromEntries( + tokens.map((token) => [ + `${token.near_token_id}:${token.defuse_asset_identifier}`, + token, + ]), + ); +} + +function mapDepositAssetId(defuseAssetIdentifier, chain) { + if (chain === config.tradingBtc.chain) return config.tradingBtc.assetId; + if (chain === config.tradingEure.chain) return config.tradingEure.assetId; + return defuseAssetIdentifier; +} diff --git a/src/apps/market-reference-ingest.mjs b/src/apps/market-reference-ingest.mjs new file mode 100644 index 0000000..c6fbd70 --- /dev/null +++ b/src/apps/market-reference-ingest.mjs @@ -0,0 +1,246 @@ +import process from 'node:process'; + +import { createProducer } from '../bus/kafka/producer.mjs'; +import { startControlApi } from '../core/control-api.mjs'; +import { buildEventEnvelope } from '../core/event-envelope.mjs'; +import { createLogger, serializeError } from '../core/log.mjs'; +import { assertMarketPriceEvent } from '../core/schemas.mjs'; +import { fetchCoinGeckoBtcEur, fetchKrakenBtcEur } from '../lib/market-data.mjs'; +import { loadConfig } from '../lib/config.mjs'; + +const config = loadConfig(); +const logger = createLogger({ + service: 'market-reference-ingest', + component: 'pricing', + namespace: config.projectNamespace, + venue: 'reference-market', +}); + +const producer = await createProducer({ + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, + logger, +}); + +const state = { + paused: false, + refreshing: false, + kraken: null, + coingecko: null, + last_published_at: null, + last_publish_error: null, + publish_count: 0, + error_count: 0, +}; + +let coingeckoDueAt = 0; + +async function refresh() { + if (state.paused || state.refreshing) return; + state.refreshing = true; + + try { + const now = Date.now(); + await refreshKraken(now).catch((error) => { + logger.warn('kraken_refresh_failed', { + pair: config.activePair, + details: { + error: serializeError(error), + }, + }); + }); + if (now >= coingeckoDueAt || !state.coingecko) { + await refreshCoinGecko(now); + coingeckoDueAt = now + config.marketReferenceCoinGeckoRefreshMs; + } + + const event = buildPriceEvent(now); + assertMarketPriceEvent(event); + await producer.sendJson(config.kafkaTopicRefMarketPrice, event, { key: event.payload.price_id }); + state.last_published_at = new Date(now).toISOString(); + state.last_publish_error = null; + state.publish_count += 1; + } catch (error) { + state.error_count += 1; + state.last_publish_error = serializeError(error); + logger.error('reference_refresh_failed', { + topic: config.kafkaTopicRefMarketPrice, + pair: config.activePair, + details: { + error: serializeError(error), + }, + }); + } finally { + state.refreshing = false; + } +} + +async function refreshKraken(now) { + try { + const price = await fetchKrakenBtcEur(config.marketReferenceKrakenTickerUrl); + state.kraken = { + price, + observed_at: new Date(now).toISOString(), + healthy: true, + error: null, + }; + } catch (error) { + state.kraken = { + ...(state.kraken || {}), + healthy: false, + error: serializeError(error), + }; + throw error; + } +} + +async function refreshCoinGecko(now) { + try { + const price = await fetchCoinGeckoBtcEur(config.marketReferenceCoinGeckoUrl); + state.coingecko = { + price, + observed_at: new Date(now).toISOString(), + healthy: true, + error: null, + }; + } catch (error) { + state.coingecko = { + ...(state.coingecko || {}), + healthy: false, + error: serializeError(error), + }; + logger.warn('coingecko_refresh_failed', { + pair: config.activePair, + details: { + error: serializeError(error), + }, + }); + } +} + +function buildPriceEvent(now) { + const sourceUsed = chooseSource(now); + if (!sourceUsed) throw new Error('No fresh reference price available'); + + const eurPerBtc = sourceUsed === 'kraken' + ? state.kraken.price + : state.coingecko.price; + const btcPerEur = 1 / eurPerBtc; + const divergencePct = state.kraken && state.coingecko + ? Math.abs((state.kraken.price - state.coingecko.price) / state.kraken.price) * 100 + : null; + + return buildEventEnvelope({ + source: 'market-reference-ingest', + venue: 'reference-market', + eventType: 'market_price', + observedAt: new Date(now).toISOString(), + payload: { + price_id: `price-${now}`, + pair: config.activePair, + eur_per_btc: eurPerBtc.toFixed(8), + eure_per_btc: eurPerBtc.toFixed(8), + btc_per_eur: btcPerEur.toFixed(12), + btc_per_eure: btcPerEur.toFixed(12), + source_used: sourceUsed, + fallback_in_use: sourceUsed !== 'kraken', + divergence_pct: divergencePct?.toFixed(6) ?? null, + eure_per_eur_assumption: '1', + kraken: compactMarketSource(state.kraken, now), + coingecko: compactMarketSource(state.coingecko, now), + }, + }); +} + +function chooseSource(now) { + if (isFresh(state.kraken, now, config.marketReferenceMaxAgeMs)) return 'kraken'; + if (isFresh(state.coingecko, now, config.marketReferenceMaxAgeMs)) return 'coingecko'; + return null; +} + +function compactMarketSource(source, now) { + if (!source) return null; + const ageMs = source.observed_at ? now - Date.parse(source.observed_at) : null; + return { + price: Number.isFinite(source.price) ? source.price.toFixed(8) : null, + observed_at: source.observed_at || null, + age_ms: Number.isFinite(ageMs) ? String(ageMs) : null, + healthy: source.healthy ?? false, + error: source.error || null, + }; +} + +function isFresh(source, now, maxAgeMs) { + if (!source?.observed_at || !Number.isFinite(source.price)) return false; + return now - Date.parse(source.observed_at) <= maxAgeMs; +} + +const timer = setInterval(refresh, config.marketReferenceRefreshMs); +timer.unref?.(); +await refresh(); + +const controlApi = startControlApi({ + host: config.marketReferenceControlHost, + port: config.marketReferenceControlPort, + logger: logger.child({ component: 'control-api' }), + service: 'market-reference-ingest', + namespace: config.projectNamespace, + stateProvider: { + getState() { + return { + ...state, + active_pair: config.activePair, + }; + }, + }, + routes: [ + { + method: 'POST', + path: '/refresh', + handler: async () => { + await refresh(); + return { + ok: true, + ...state, + }; + }, + }, + { + method: 'POST', + path: '/pause', + handler: () => { + state.paused = true; + logger.warn('polling_paused', { + pair: config.activePair, + }); + return { ok: true, paused: true }; + }, + }, + { + method: 'POST', + path: '/resume', + handler: async () => { + state.paused = false; + logger.info('polling_resumed', { + pair: config.activePair, + }); + await refresh(); + return { ok: true, paused: false }; + }, + }, + ], +}); + +async function shutdown() { + clearInterval(timer); + await controlApi.close().catch(() => {}); + await producer.disconnect(); + process.exit(0); +} + +process.on('SIGINT', shutdown); +process.on('SIGTERM', shutdown); + +function compact(value) { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry != null)); +} diff --git a/src/apps/near-intents-ingest.mjs b/src/apps/near-intents-ingest.mjs index 565f96d..def2a53 100644 --- a/src/apps/near-intents-ingest.mjs +++ b/src/apps/near-intents-ingest.mjs @@ -65,8 +65,47 @@ const controlApi = config.nearIntentsControlApiEnabled }), service: 'near-intents-ingest', namespace: config.projectNamespace, - pairFilterController, - stateProvider: wsRuntime, + stateProvider: { + getState() { + return { + pair_filter: pairFilterController.getState(), + ingest: wsRuntime.getState(), + }; + }, + }, + routes: [ + { + method: 'GET', + path: '/pair-filter', + readBody: false, + handler: () => pairFilterController.getState(), + }, + { + method: 'PUT', + path: '/pair-filter', + handler: ({ body }) => { + if (body.disabled === true || body.enabled === false || body.pair == null) { + return pairFilterController.disable(); + } + + if (typeof body.pair !== 'string') { + return { + statusCode: 400, + payload: { + error: 'send JSON like {"pair":"asset_a->asset_b"} or {"pair":null}', + }, + }; + } + + return pairFilterController.setPairFilter(body.pair); + }, + }, + { + method: 'POST', + path: '/pair-filter/reset', + handler: () => pairFilterController.reset(), + }, + ], }) : null; diff --git a/src/apps/strategy-engine.mjs b/src/apps/strategy-engine.mjs new file mode 100644 index 0000000..90b6e8c --- /dev/null +++ b/src/apps/strategy-engine.mjs @@ -0,0 +1,233 @@ +import process from 'node:process'; + +import { createConsumer } from '../bus/kafka/consumer.mjs'; +import { createProducer } from '../bus/kafka/producer.mjs'; +import { startControlApi } from '../core/control-api.mjs'; +import { buildEventEnvelope, parseEventMessage } from '../core/event-envelope.mjs'; +import { createLogger, serializeError } from '../core/log.mjs'; +import { assertInventorySnapshotEvent, assertMarketPriceEvent, assertNormalizedSwapDemand } from '../core/schemas.mjs'; +import { evaluateTradeOpportunity } from '../core/strategy.mjs'; +import { loadConfig } from '../lib/config.mjs'; + +const config = loadConfig(); +const logger = createLogger({ + service: 'strategy-engine', + component: 'strategy', + namespace: config.projectNamespace, + venue: 'near-intents', +}); + +const consumer = await createConsumer({ + groupId: config.kafkaConsumerGroupStrategy, + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, + logger, +}); +const producer = await createProducer({ + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, + logger, +}); + +await consumer.subscribe({ topic: config.kafkaTopicNormSwapDemand, fromBeginning: false }); +await consumer.subscribe({ topic: config.kafkaTopicRefMarketPrice, fromBeginning: false }); +await consumer.subscribe({ topic: config.kafkaTopicStateIntentInventory, fromBeginning: false }); + +const state = { + armed: config.strategyInitialArmed, + paused: false, + threshold_pct: config.strategyGrossThresholdPct, + max_notional_eure: config.strategyMaxNotionalEure, + latest_price_event: null, + latest_inventory_event: null, + latest_decision: null, + recent_decisions: [], + skipped_counts: {}, + seen_quotes: {}, +}; + +await consumer.run({ + eachMessage: async ({ topic, message }) => { + if (!message.value) return; + + try { + const event = parseEventMessage(message.value.toString()); + + if (topic === config.kafkaTopicRefMarketPrice) { + assertMarketPriceEvent(event); + state.latest_price_event = event; + return; + } + + if (topic === config.kafkaTopicStateIntentInventory) { + assertInventorySnapshotEvent(event); + state.latest_inventory_event = event; + return; + } + + assertNormalizedSwapDemand(event); + await handleDemand(event); + } catch (error) { + logger.error('strategy_message_failed', { + topic, + pair: config.activePair, + details: { + error: serializeError(error), + }, + }); + } + }, +}); + +async function handleDemand(event) { + if (state.paused) return; + + if (state.seen_quotes[event.payload.quote_id]) { + await publishDecision({ + decision_id: `duplicate-${event.payload.quote_id}`, + quote_id: event.payload.quote_id, + pair: event.payload.pair || `${event.payload.asset_in}->${event.payload.asset_out}`, + direction: 'duplicate', + request_kind: event.payload.request_kind, + decision: 'rejected', + decision_reason: 'duplicate_quote_id', + threshold_pct: String(state.threshold_pct), + max_notional_eure: String(state.max_notional_eure), + strategy_armed: state.armed, + }); + return; + } + + state.seen_quotes[event.payload.quote_id] = true; + + const evaluation = evaluateTradeOpportunity({ + demandEvent: event, + priceEvent: state.latest_price_event, + inventoryEvent: state.latest_inventory_event, + config, + armed: state.armed, + thresholdPct: state.threshold_pct, + maxNotionalEure: state.max_notional_eure, + }); + + await publishDecision(evaluation.decision); + + if (evaluation.command) { + const commandEvent = buildEventEnvelope({ + source: 'strategy-engine', + venue: 'near-intents', + eventType: 'execute_trade', + observedAt: event.observed_at, + payload: evaluation.command, + }); + await producer.sendJson(config.kafkaTopicCmdExecuteTrade, commandEvent, { key: evaluation.command.execution_key }); + } +} + +async function publishDecision(decisionPayload) { + const event = buildEventEnvelope({ + source: 'strategy-engine', + venue: 'near-intents', + eventType: 'trade_decision', + payload: decisionPayload, + }); + await producer.sendJson(config.kafkaTopicDecisionTradeDecision, event, { key: decisionPayload.quote_id }); + state.latest_decision = decisionPayload; + state.recent_decisions.unshift(decisionPayload); + state.recent_decisions = state.recent_decisions.slice(0, 20); + state.skipped_counts[decisionPayload.decision_reason] = + (state.skipped_counts[decisionPayload.decision_reason] || 0) + 1; +} + +const controlApi = startControlApi({ + host: config.strategyEngineControlHost, + port: config.strategyEngineControlPort, + logger: logger.child({ component: 'control-api' }), + service: 'strategy-engine', + namespace: config.projectNamespace, + stateProvider: { + getState() { + return state; + }, + }, + routes: [ + { + method: 'POST', + path: '/arm', + handler: () => { + state.armed = true; + logger.warn('strategy_armed', { pair: config.activePair }); + return { ok: true, armed: true }; + }, + }, + { + method: 'POST', + path: '/disarm', + handler: () => { + state.armed = false; + logger.warn('strategy_disarmed', { pair: config.activePair }); + return { ok: true, armed: false }; + }, + }, + { + method: 'POST', + path: '/pause', + handler: () => { + state.paused = true; + consumer.pause([ + { topic: config.kafkaTopicNormSwapDemand }, + { topic: config.kafkaTopicRefMarketPrice }, + { topic: config.kafkaTopicStateIntentInventory }, + ]); + return { ok: true, paused: true }; + }, + }, + { + method: 'POST', + path: '/resume', + handler: () => { + state.paused = false; + consumer.resume([ + { topic: config.kafkaTopicNormSwapDemand }, + { topic: config.kafkaTopicRefMarketPrice }, + { topic: config.kafkaTopicStateIntentInventory }, + ]); + return { ok: true, paused: false }; + }, + }, + { + method: 'PUT', + path: '/threshold', + handler: ({ body }) => { + const next = Number(body.threshold_pct); + if (!Number.isFinite(next) || next <= 0) { + return { statusCode: 400, payload: { error: 'threshold_pct must be > 0' } }; + } + state.threshold_pct = next; + return { ok: true, threshold_pct: next }; + }, + }, + { + method: 'PUT', + path: '/limits', + handler: ({ body }) => { + const next = Number(body.max_notional_eure); + if (!Number.isFinite(next) || next <= 0) { + return { statusCode: 400, payload: { error: 'max_notional_eure must be > 0' } }; + } + state.max_notional_eure = next; + return { ok: true, max_notional_eure: next }; + }, + }, + ], +}); + +async function shutdown() { + await controlApi.close().catch(() => {}); + await consumer.disconnect(); + await producer.disconnect(); + process.exit(0); +} + +process.on('SIGINT', shutdown); +process.on('SIGTERM', shutdown); diff --git a/src/apps/trade-executor.mjs b/src/apps/trade-executor.mjs new file mode 100644 index 0000000..9f4c2f2 --- /dev/null +++ b/src/apps/trade-executor.mjs @@ -0,0 +1,277 @@ +import process from 'node:process'; + +import { createConsumer } from '../bus/kafka/consumer.mjs'; +import { createProducer } from '../bus/kafka/producer.mjs'; +import { startControlApi } from '../core/control-api.mjs'; +import { buildEventEnvelope, parseEventMessage } from '../core/event-envelope.mjs'; +import { createExecutorStateStore } from '../core/executor-state-store.mjs'; +import { createLogger, serializeError } from '../core/log.mjs'; +import { assertExecuteTradeCommand, assertTradeResult } from '../core/schemas.mjs'; +import { loadConfig } from '../lib/config.mjs'; +import { buildQuoteResponseSubmission } from '../venues/near-intents/signing.mjs'; +import { startSolverRelayWs } from '../venues/near-intents/solver-relay-ws.mjs'; +import { createVerifierClient } from '../venues/near-intents/verifier-client.mjs'; + +const config = loadConfig(); +const logger = createLogger({ + service: 'trade-executor', + component: 'executor', + namespace: config.projectNamespace, + venue: 'near-intents', +}); + +if (!config.nearIntentsApiKey || !config.nearIntentsAccountId || !config.nearIntentsSignerPrivateKey) { + logger.error('missing_executor_config', { + details: { + required: [ + 'NEAR_INTENTS_API_KEY', + 'NEAR_INTENTS_ACCOUNT_ID', + 'NEAR_INTENTS_SIGNER_PRIVATE_KEY', + ], + }, + }); + process.exit(1); +} + +const consumer = await createConsumer({ + groupId: config.kafkaConsumerGroupExecutor, + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, + logger, +}); +const producer = await createProducer({ + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, + logger, +}); + +const verifierClient = createVerifierClient({ + nearRpcUrl: config.nearRpcUrl, + verifierContract: config.nearVerifierContract, + signerPrivateKey: config.nearIntentsSignerPrivateKey, +}); +const signer = verifierClient.getSigner(); +const relayClient = await startSolverRelayWs({ + apiKey: config.nearIntentsApiKey, + wsUrl: config.nearIntentsWsUrl, + logger: logger.child({ component: 'solver-relay' }), + subscriptions: ['quote_status'], + onEvent(payload) { + state.last_quote_status = payload?.params || payload?.result || payload; + }, +}); +const stateStore = createExecutorStateStore({ stateDir: config.executorStateDir }); + +const state = { + armed: config.executorInitialArmed, + paused: false, + draining: false, + last_command: null, + last_request: null, + last_venue_response: null, + last_quote_status: null, + last_error: null, + in_flight_count: 0, + completed_count: 0, +}; + +await consumer.subscribe({ topic: config.kafkaTopicCmdExecuteTrade, fromBeginning: false }); +await consumer.run({ + eachMessage: async ({ message }) => { + if (!message.value) return; + try { + const event = parseEventMessage(message.value.toString()); + assertExecuteTradeCommand(event); + await handleCommand(event); + } catch (error) { + state.last_error = serializeError(error); + logger.error('executor_message_failed', { + topic: config.kafkaTopicCmdExecuteTrade, + details: { + error: serializeError(error), + }, + }); + } + }, +}); + +async function handleCommand(event) { + const payload = event.payload; + state.last_command = payload; + + const existing = stateStore.get(payload.command_id); + if (existing?.status === 'completed') { + logger.warn('duplicate_command_skipped', { + topic: config.kafkaTopicCmdExecuteTrade, + pair: payload.pair, + details: { + command_id: payload.command_id, + }, + }); + return; + } + + if (state.paused) return; + + if (!state.armed) { + await publishResult(payload, { + status: 'rejected', + result_code: 'executor_disarmed', + note: 'executor is disarmed', + }); + return; + } + + stateStore.markProcessing(payload.command_id, { + quote_id: payload.quote_id, + idempotency_key: payload.idempotency_key, + execution_key: payload.execution_key, + }); + state.in_flight_count += 1; + + try { + const currentSaltHex = await verifierClient.currentSalt(); + const submission = buildQuoteResponseSubmission({ + command: payload, + signerAccountId: config.nearIntentsAccountId, + signer, + verifierContract: config.nearVerifierContract, + currentSaltHex, + }); + state.last_request = submission; + + const response = await relayClient.request('quote_response', [submission], { + timeoutMs: config.executorResponseTimeoutMs, + }); + state.last_venue_response = response; + state.last_error = null; + + await publishResult(payload, { + status: 'submitted', + result_code: response === 'OK' ? 'quote_response_ok' : 'quote_response_ack', + venue_response: response, + }); + stateStore.markCompleted(payload.command_id, { + quote_id: payload.quote_id, + result: response, + }); + state.completed_count += 1; + } catch (error) { + state.last_error = serializeError(error); + stateStore.markFailed(payload.command_id, { + quote_id: payload.quote_id, + error: serializeError(error), + }); + await publishResult(payload, { + status: 'failed', + result_code: 'submission_failed', + error: serializeError(error), + }); + } finally { + state.in_flight_count = Math.max(0, state.in_flight_count - 1); + if (state.draining && state.in_flight_count === 0) { + setTimeout(() => shutdown(), 0); + } + } +} + +async function publishResult(command, extraPayload) { + const event = buildEventEnvelope({ + source: 'trade-executor', + venue: 'near-intents', + eventType: 'trade_result', + payload: { + command_id: command.command_id, + decision_id: command.decision_id, + idempotency_key: command.idempotency_key, + execution_key: command.execution_key, + quote_id: command.quote_id, + pair: command.pair, + ...extraPayload, + }, + }); + assertTradeResult(event); + await producer.sendJson(config.kafkaTopicExecTradeResult, event, { key: command.execution_key }); +} + +const controlApi = startControlApi({ + host: config.tradeExecutorControlHost, + port: config.tradeExecutorControlPort, + logger: logger.child({ component: 'control-api' }), + service: 'trade-executor', + namespace: config.projectNamespace, + stateProvider: { + async getState() { + const signerRegistered = await verifierClient.isPublicKeyRegistered({ + accountId: config.nearIntentsAccountId, + }).catch(() => null); + return { + account_id: config.nearIntentsAccountId, + signer_public_key: signer.getPublicKey().toString(), + signer_registered: signerRegistered, + ...state, + durable_state: stateStore.getState(), + }; + }, + }, + routes: [ + { + method: 'POST', + path: '/arm', + handler: () => { + state.armed = true; + return { ok: true, armed: true }; + }, + }, + { + method: 'POST', + path: '/disarm', + handler: () => { + state.armed = false; + return { ok: true, armed: false }; + }, + }, + { + method: 'POST', + path: '/pause', + handler: () => { + state.paused = true; + consumer.pause([{ topic: config.kafkaTopicCmdExecuteTrade }]); + return { ok: true, paused: true }; + }, + }, + { + method: 'POST', + path: '/resume', + handler: () => { + state.paused = false; + consumer.resume([{ topic: config.kafkaTopicCmdExecuteTrade }]); + return { ok: true, paused: false }; + }, + }, + { + method: 'POST', + path: '/drain', + handler: () => { + state.draining = true; + state.paused = true; + consumer.pause([{ topic: config.kafkaTopicCmdExecuteTrade }]); + if (state.in_flight_count === 0) { + setTimeout(() => shutdown(), 0); + } + return { ok: true, draining: true }; + }, + }, + ], +}); + +async function shutdown() { + await controlApi.close().catch(() => {}); + relayClient.close(); + await consumer.disconnect(); + await producer.disconnect(); + process.exit(0); +} + +process.on('SIGINT', shutdown); +process.on('SIGTERM', shutdown); diff --git a/src/bus/kafka/consumer.mjs b/src/bus/kafka/consumer.mjs index 09b6829..dcd58c1 100644 --- a/src/bus/kafka/consumer.mjs +++ b/src/bus/kafka/consumer.mjs @@ -47,6 +47,9 @@ export async function createConsumer({ groupId, logger, ...options }) { return { subscribe: (options) => consumer.subscribe(options), run: (options) => consumer.run(options), + pause: (topics) => consumer.pause(topics), + resume: (topics) => consumer.resume(topics), + stop: () => consumer.stop(), disconnect: () => consumer.disconnect(), }; } diff --git a/src/core/assets.mjs b/src/core/assets.mjs new file mode 100644 index 0000000..23845c5 --- /dev/null +++ b/src/core/assets.mjs @@ -0,0 +1,55 @@ +export function pairKey(assetIn, assetOut) { + return `${assetIn}->${assetOut}`; +} + +export function classifyPairDirection({ assetIn, assetOut, btcAssetId, eureAssetId }) { + if (assetIn === btcAssetId && assetOut === eureAssetId) return 'btc_to_eure'; + if (assetIn === eureAssetId && assetOut === btcAssetId) return 'eure_to_btc'; + return 'unsupported'; +} + +export function isActivePair(assetIn, assetOut, config) { + return ( + (assetIn === config.tradingBtc.assetId && assetOut === config.tradingEure.assetId) + || (assetIn === config.tradingEure.assetId && assetOut === config.tradingBtc.assetId) + ); +} + +export function unitsToNumber(amount, decimals) { + if (amount == null) return 0; + const raw = String(amount).trim(); + if (!raw) return 0; + + const negative = raw.startsWith('-'); + const digits = negative ? raw.slice(1) : raw; + const padded = digits.padStart(decimals + 1, '0'); + const head = padded.slice(0, padded.length - decimals); + const tail = decimals > 0 ? padded.slice(-decimals) : ''; + const rendered = decimals > 0 ? `${head}.${tail}` : head; + const numeric = Number(rendered); + return negative ? -numeric : numeric; +} + +export function numberToUnits(value, decimals, { mode = 'round' } = {}) { + const factor = 10 ** decimals; + const scaled = value * factor; + + if (mode === 'floor') return String(Math.max(0, Math.floor(scaled))); + if (mode === 'ceil') return String(Math.max(0, Math.ceil(scaled))); + return String(Math.max(0, Math.round(scaled))); +} + +export function bigintAmount(amount) { + return BigInt(String(amount || '0')); +} + +export function formatNumber(value, digits = 8) { + if (!Number.isFinite(value)) return null; + return value.toFixed(digits); +} + +export function mapToSortedObject(record = {}) { + return Object.fromEntries( + Object.entries(record).sort(([left], [right]) => left.localeCompare(right)), + ); +} diff --git a/src/core/control-api.mjs b/src/core/control-api.mjs index d312ebb..9c863db 100644 --- a/src/core/control-api.mjs +++ b/src/core/control-api.mjs @@ -4,11 +4,14 @@ export function startControlApi({ host = '0.0.0.0', port = 8081, logger = null, - service = 'near-intents-ingest', + service = 'service', namespace = 'unrip', - pairFilterController, stateProvider = null, + healthProvider = null, + routes = [], } = {}) { + const routeMap = new Map(routes.map((route) => [`${route.method} ${route.path}`, route])); + const server = http.createServer(async (req, res) => { try { if (req.method === 'GET' && req.url === '/healthz') { @@ -16,61 +19,37 @@ export function startControlApi({ ok: true, service, namespace, + ...(await healthProvider?.getHealth?.()), }); } if (req.method === 'GET' && req.url === '/state') { - return sendJson(res, 200, buildStateResponse({ - service, - namespace, - pairFilterController, - stateProvider, - })); - } - - if (req.method === 'GET' && req.url === '/pair-filter') { return sendJson(res, 200, { service, namespace, - ...pairFilterController.getState(), + ...(await stateProvider?.getState?.()), }); } - if (req.method === 'PUT' && req.url === '/pair-filter') { - const body = await readJsonBody(req); - - if (body.disabled === true || body.enabled === false || body.pair == null) { - return sendJson(res, 200, { - service, - namespace, - ...pairFilterController.disable(), - }); - } - - if (typeof body.pair !== 'string') { - return sendJson(res, 400, { - error: "send JSON like {\"pair\":\"asset_a->asset_b\"} or {\"pair\":null}", - }); - } - - return sendJson(res, 200, { - service, - namespace, - ...pairFilterController.setPairFilter(body.pair), + const route = routeMap.get(`${req.method} ${req.url}`); + if (!route) { + return sendJson(res, 404, { + error: 'not_found', }); } - if (req.method === 'POST' && req.url === '/pair-filter/reset') { - return sendJson(res, 200, { - service, - namespace, - ...pairFilterController.reset(), - }); - } - - return sendJson(res, 404, { - error: 'not_found', + const body = route.readBody === false ? null : await readJsonBody(req); + const result = await route.handler({ + req, + res, + body: body || {}, }); + + if (result == null) return; + if (result.statusCode != null) { + return sendJson(res, result.statusCode, result.payload ?? {}); + } + return sendJson(res, 200, result); } catch (error) { logger?.error('control_api_request_failed', { details: { @@ -106,28 +85,14 @@ export function startControlApi({ }; } -function buildStateResponse({ - service, - namespace, - pairFilterController, - stateProvider, -}) { - return { - service, - namespace, - pair_filter: pairFilterController.getState(), - ingest: stateProvider?.getState?.() ?? null, - }; -} - -function sendJson(res, statusCode, payload) { +export function sendJson(res, statusCode, payload) { const body = JSON.stringify(payload, null, 2); res.statusCode = statusCode; res.setHeader('content-type', 'application/json; charset=utf-8'); res.end(`${body}\n`); } -function readJsonBody(req) { +export function readJsonBody(req) { return new Promise((resolve, reject) => { let raw = ''; diff --git a/src/core/executor-state-store.mjs b/src/core/executor-state-store.mjs index b208c10..726126b 100644 --- a/src/core/executor-state-store.mjs +++ b/src/core/executor-state-store.mjs @@ -1,49 +1,45 @@ -import fs from 'node:fs'; -import path from 'node:path'; +import { createJsonStateStore } from './json-state-store.mjs'; + +const INITIAL_STATE = { + commands: {}, +}; export function createExecutorStateStore({ stateDir, fileName = 'commands.json' }) { - fs.mkdirSync(stateDir, { recursive: true }); - const filePath = path.join(stateDir, fileName); - const state = loadState(filePath); + const store = createJsonStateStore({ + stateDir, + fileName, + initialState: INITIAL_STATE, + }); return { get(commandId) { - return state[commandId] || null; + return store.getState().commands[commandId] || null; }, markProcessing(commandId, metadata) { - state[commandId] = { - ...(state[commandId] || {}), - ...metadata, - status: 'processing', - updated_at: new Date().toISOString(), - }; - persistState(filePath, state); - return state[commandId]; + return updateCommand(store, commandId, metadata, 'processing'); }, markCompleted(commandId, metadata) { - state[commandId] = { - ...(state[commandId] || {}), - ...metadata, - status: 'completed', - updated_at: new Date().toISOString(), - }; - persistState(filePath, state); - return state[commandId]; + return updateCommand(store, commandId, metadata, 'completed'); + }, + markFailed(commandId, metadata) { + return updateCommand(store, commandId, metadata, 'failed'); + }, + getState() { + return store.getState(); }, }; } -function loadState(filePath) { - if (!fs.existsSync(filePath)) return {}; - try { - return JSON.parse(fs.readFileSync(filePath, 'utf8')); - } catch { - return {}; - } -} +function updateCommand(store, commandId, metadata, status) { + const nextState = store.update((state) => { + state.commands[commandId] = { + ...(state.commands[commandId] || {}), + ...metadata, + status, + updated_at: new Date().toISOString(), + }; + return state; + }); -function persistState(filePath, state) { - const tempPath = `${filePath}.tmp`; - fs.writeFileSync(tempPath, JSON.stringify(state, null, 2)); - fs.renameSync(tempPath, filePath); + return nextState.commands[commandId]; } diff --git a/src/core/history-records.mjs b/src/core/history-records.mjs new file mode 100644 index 0000000..0bae447 --- /dev/null +++ b/src/core/history-records.mjs @@ -0,0 +1,126 @@ +import { + assertExecuteTradeCommand, + assertInventorySnapshotEvent, + assertLiquidityActionEvent, + assertMarketPriceEvent, + assertNormalizedSwapDemand, + assertTradeDecisionEvent, + assertTradeResult, +} from './schemas.mjs'; + +export function routeHistoryRecord({ topic, event }) { + switch (topic) { + case 'raw.near_intents.quote': + return { + table: 'raw_near_intents_quotes', + record: { + event_id: event.event_id, + observed_at: event.observed_at, + ingested_at: event.ingested_at, + quote_id: event.payload?.message?.quote_id || event.payload?.message?.quote_hash || null, + pair: buildPairFromMessage(event.payload?.message), + }, + }; + case 'norm.swap_demand': + assertNormalizedSwapDemand(event); + return { + table: 'swap_demand_events', + record: { + event_id: event.event_id, + observed_at: event.observed_at, + ingested_at: event.ingested_at, + quote_id: event.payload.quote_id, + pair: `${event.payload.asset_in}->${event.payload.asset_out}`, + decision_key: null, + }, + }; + case 'ref.market_price': + assertMarketPriceEvent(event); + return { + table: 'market_price_events', + record: { + event_id: event.event_id, + observed_at: event.observed_at, + ingested_at: event.ingested_at, + quote_id: null, + pair: event.payload.pair, + decision_key: event.payload.price_id, + }, + }; + case 'state.intent_inventory': + assertInventorySnapshotEvent(event); + return { + table: 'intent_inventory_snapshots', + record: { + event_id: event.event_id, + observed_at: event.observed_at, + ingested_at: event.ingested_at, + quote_id: null, + pair: null, + decision_key: event.payload.inventory_id, + }, + }; + case 'ops.liquidity_action': + assertLiquidityActionEvent(event); + return { + table: 'liquidity_actions', + record: { + event_id: event.event_id, + observed_at: event.observed_at, + ingested_at: event.ingested_at, + quote_id: null, + pair: null, + decision_key: event.payload.liquidity_action_id, + }, + }; + case 'decision.trade_decision': + assertTradeDecisionEvent(event); + return { + table: 'trade_decisions', + record: { + event_id: event.event_id, + observed_at: event.observed_at, + ingested_at: event.ingested_at, + quote_id: event.payload.quote_id, + pair: event.payload.pair, + decision_key: event.payload.decision_id, + }, + }; + case 'cmd.execute_trade': + assertExecuteTradeCommand(event); + return { + table: 'execute_trade_commands', + record: { + event_id: event.event_id, + observed_at: event.observed_at, + ingested_at: event.ingested_at, + quote_id: event.payload.quote_id, + pair: event.payload.pair, + decision_key: event.payload.command_id, + }, + }; + case 'exec.trade_result': + assertTradeResult(event); + return { + table: 'trade_execution_results', + record: { + event_id: event.event_id, + observed_at: event.observed_at, + ingested_at: event.ingested_at, + quote_id: event.payload.quote_id, + pair: event.payload.pair || null, + decision_key: event.payload.command_id, + }, + }; + default: + throw new Error(`Unsupported topic: ${topic}`); + } +} + +function buildPairFromMessage(message) { + if (!message) return null; + const assetIn = message.defuse_asset_identifier_in || message.asset_in || null; + const assetOut = message.defuse_asset_identifier_out || message.asset_out || null; + if (!assetIn || !assetOut) return null; + return `${assetIn}->${assetOut}`; +} diff --git a/src/core/inventory.mjs b/src/core/inventory.mjs new file mode 100644 index 0000000..5e4a933 --- /dev/null +++ b/src/core/inventory.mjs @@ -0,0 +1,67 @@ +import crypto from 'node:crypto'; + +import { mapToSortedObject } from './assets.mjs'; + +export function buildInventorySnapshot({ + accountId, + balances, + recentDeposits, + trackedWithdrawals, + assetRegistry, + observedAt = new Date().toISOString(), +}) { + const spendable = {}; + for (const assetId of assetRegistry.keys()) { + spendable[assetId] = String(balances[assetId] || '0'); + } + + const pendingInbound = {}; + const pendingOutbound = {}; + const depositRecords = []; + const withdrawalRecords = []; + + for (const [assetId] of assetRegistry) { + pendingInbound[assetId] = '0'; + pendingOutbound[assetId] = '0'; + } + + for (const deposit of recentDeposits) { + depositRecords.push(deposit); + if (deposit.status === 'PENDING') { + const assetId = deposit.asset_id; + pendingInbound[assetId] = addStrings(pendingInbound[assetId], deposit.amount); + } + } + + for (const withdrawal of trackedWithdrawals) { + withdrawalRecords.push(withdrawal); + if (withdrawal.status && withdrawal.status !== 'COMPLETED' && withdrawal.asset_id) { + pendingOutbound[withdrawal.asset_id] = addStrings( + pendingOutbound[withdrawal.asset_id], + withdrawal.amount || '0', + ); + } + } + + return { + inventory_id: crypto.randomUUID(), + account_id: accountId, + synced_at: observedAt, + spendable: mapToSortedObject(spendable), + pending_inbound: mapToSortedObject(pendingInbound), + pending_outbound: mapToSortedObject(pendingOutbound), + deposits: depositRecords, + withdrawals: withdrawalRecords, + reconciliation_status: hasPending(pendingInbound) || hasPending(pendingOutbound) + ? 'pending_transfers' + : 'ok', + }; +} + +function addStrings(left, right) { + return (BigInt(left || '0') + BigInt(right || '0')).toString(); +} + +function hasPending(values) { + return Object.values(values).some((value) => BigInt(value || '0') > 0n); +} diff --git a/src/core/json-state-store.mjs b/src/core/json-state-store.mjs new file mode 100644 index 0000000..c688559 --- /dev/null +++ b/src/core/json-state-store.mjs @@ -0,0 +1,52 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +export function createJsonStateStore({ stateDir, fileName, initialState = {} }) { + if (!stateDir) throw new Error('Missing stateDir'); + if (!fileName) throw new Error('Missing fileName'); + + fs.mkdirSync(stateDir, { recursive: true }); + const filePath = path.join(stateDir, fileName); + const state = loadState(filePath, initialState); + + return { + getState() { + return state; + }, + setState(nextState) { + replaceState(state, nextState); + persistState(filePath, state); + return state; + }, + update(mutator) { + const nextState = mutator(structuredClone(state)); + replaceState(state, nextState); + persistState(filePath, state); + return state; + }, + }; +} + +function replaceState(target, nextState) { + for (const key of Object.keys(target)) delete target[key]; + Object.assign(target, nextState || {}); +} + +function loadState(filePath, initialState) { + if (!fs.existsSync(filePath)) return structuredClone(initialState); + + try { + return { + ...structuredClone(initialState), + ...JSON.parse(fs.readFileSync(filePath, 'utf8')), + }; + } catch { + return structuredClone(initialState); + } +} + +function persistState(filePath, state) { + const tempPath = `${filePath}.tmp`; + fs.writeFileSync(tempPath, JSON.stringify(state, null, 2)); + fs.renameSync(tempPath, filePath); +} diff --git a/src/core/schemas.mjs b/src/core/schemas.mjs index 50ca90d..734f5bd 100644 --- a/src/core/schemas.mjs +++ b/src/core/schemas.mjs @@ -6,6 +6,10 @@ function requireObject(value, field) { if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`Missing ${field}`); } +function requireOneOf(value, field, values) { + if (!values.includes(value)) throw new Error(`Unexpected ${field}: ${value}`); +} + export function assertEventEnvelope(event) { requireObject(event, 'event'); requireString(event.event_id, 'event.event_id'); @@ -26,9 +30,61 @@ export function assertNormalizedSwapDemand(event) { requireString(payload.quote_id, 'payload.quote_id'); requireString(payload.asset_in, 'payload.asset_in'); requireString(payload.asset_out, 'payload.asset_out'); + requireString(payload.request_kind, 'payload.request_kind'); + requireOneOf(payload.request_kind, 'payload.request_kind', ['exact_in', 'exact_out']); if (payload.amount_in != null) requireString(payload.amount_in, 'payload.amount_in'); if (payload.amount_out != null) requireString(payload.amount_out, 'payload.amount_out'); - if (payload.ttl_ms != null) requireString(payload.ttl_ms, 'payload.ttl_ms'); + if (payload.min_deadline_ms != null) requireString(payload.min_deadline_ms, 'payload.min_deadline_ms'); + return event; +} + +export function assertMarketPriceEvent(event) { + assertEventEnvelope(event); + if (event.event_type !== 'market_price') throw new Error(`Unexpected event_type: ${event.event_type}`); + + const payload = event.payload; + requireString(payload.price_id, 'payload.price_id'); + requireString(payload.pair, 'payload.pair'); + requireString(payload.eur_per_btc, 'payload.eur_per_btc'); + requireString(payload.btc_per_eure, 'payload.btc_per_eure'); + requireString(payload.source_used, 'payload.source_used'); + return event; +} + +export function assertInventorySnapshotEvent(event) { + assertEventEnvelope(event); + if (event.event_type !== 'intent_inventory') throw new Error(`Unexpected event_type: ${event.event_type}`); + + const payload = event.payload; + requireString(payload.inventory_id, 'payload.inventory_id'); + requireString(payload.account_id, 'payload.account_id'); + requireString(payload.reconciliation_status, 'payload.reconciliation_status'); + requireObject(payload.spendable, 'payload.spendable'); + return event; +} + +export function assertLiquidityActionEvent(event) { + assertEventEnvelope(event); + if (event.event_type !== 'liquidity_action') throw new Error(`Unexpected event_type: ${event.event_type}`); + + const payload = event.payload; + requireString(payload.liquidity_action_id, 'payload.liquidity_action_id'); + requireString(payload.action_type, 'payload.action_type'); + requireString(payload.status, 'payload.status'); + return event; +} + +export function assertTradeDecisionEvent(event) { + assertEventEnvelope(event); + if (event.event_type !== 'trade_decision') throw new Error(`Unexpected event_type: ${event.event_type}`); + + const payload = event.payload; + requireString(payload.decision_id, 'payload.decision_id'); + requireString(payload.quote_id, 'payload.quote_id'); + requireString(payload.pair, 'payload.pair'); + requireString(payload.direction, 'payload.direction'); + requireString(payload.decision, 'payload.decision'); + requireString(payload.decision_reason, 'payload.decision_reason'); return event; } @@ -38,11 +94,14 @@ export function assertExecuteTradeCommand(event) { const payload = event.payload; requireString(payload.command_id, 'payload.command_id'); + requireString(payload.decision_id, 'payload.decision_id'); requireString(payload.idempotency_key, 'payload.idempotency_key'); requireString(payload.execution_key, 'payload.execution_key'); requireString(payload.quote_id, 'payload.quote_id'); requireString(payload.asset_in, 'payload.asset_in'); requireString(payload.asset_out, 'payload.asset_out'); + requireString(payload.request_kind, 'payload.request_kind'); + requireObject(payload.quote_output, 'payload.quote_output'); if (payload.amount_in != null) requireString(payload.amount_in, 'payload.amount_in'); if (payload.amount_out != null) requireString(payload.amount_out, 'payload.amount_out'); return event; @@ -54,6 +113,7 @@ export function assertTradeResult(event) { const payload = event.payload; requireString(payload.command_id, 'payload.command_id'); + requireString(payload.decision_id, 'payload.decision_id'); requireString(payload.idempotency_key, 'payload.idempotency_key'); requireString(payload.execution_key, 'payload.execution_key'); requireString(payload.quote_id, 'payload.quote_id'); diff --git a/src/core/strategy.mjs b/src/core/strategy.mjs new file mode 100644 index 0000000..80564e8 --- /dev/null +++ b/src/core/strategy.mjs @@ -0,0 +1,333 @@ +import crypto from 'node:crypto'; + +import { + bigintAmount, + classifyPairDirection, + formatNumber, + isActivePair, + numberToUnits, + pairKey, + unitsToNumber, +} from './assets.mjs'; + +export function evaluateTradeOpportunity({ + demandEvent, + priceEvent, + inventoryEvent, + config, + now = Date.now(), + armed = false, + thresholdPct = config.strategyGrossThresholdPct, + maxNotionalEure = config.strategyMaxNotionalEure, +}) { + const payload = demandEvent.payload; + const decisionId = crypto.randomUUID(); + const baseDecision = { + decision_id: decisionId, + quote_id: payload.quote_id, + pair: pairKey(payload.asset_in, payload.asset_out), + direction: classifyPairDirection({ + assetIn: payload.asset_in, + assetOut: payload.asset_out, + btcAssetId: config.tradingBtc.assetId, + eureAssetId: config.tradingEure.assetId, + }), + request_kind: payload.request_kind, + decision: 'rejected', + decision_reason: 'unknown', + threshold_pct: String(thresholdPct), + max_notional_eure: String(maxNotionalEure), + strategy_armed: armed, + assumptions: { + eure_per_eur: '1', + }, + }; + + if (!isActivePair(payload.asset_in, payload.asset_out, config)) { + return { decision: withReason(baseDecision, 'unsupported_pair') }; + } + + if (!priceEvent) { + return { decision: withReason(baseDecision, 'no_reference_price') }; + } + + if (!inventoryEvent) { + return { decision: withReason(baseDecision, 'no_inventory_snapshot') }; + } + + const priceAgeMs = now - Date.parse(priceEvent.ingested_at || priceEvent.observed_at || 0); + const inventoryAgeMs = now - Date.parse(inventoryEvent.ingested_at || inventoryEvent.observed_at || 0); + + if (!Number.isFinite(priceAgeMs) || priceAgeMs > config.strategyPriceMaxAgeMs) { + return { + decision: { + ...withReason(baseDecision, 'stale_reference_price'), + price_freshness_ms: String(priceAgeMs), + }, + }; + } + + if (!Number.isFinite(inventoryAgeMs) || inventoryAgeMs > config.strategyInventoryMaxAgeMs) { + return { + decision: { + ...withReason(baseDecision, 'stale_inventory_snapshot'), + inventory_freshness_ms: String(inventoryAgeMs), + }, + }; + } + + const price = priceEvent.payload; + const inventory = inventoryEvent.payload; + const buildResult = buildQuote({ + demand: payload, + price, + inventory, + config, + thresholdPct, + maxNotionalEure, + }); + + if (!buildResult.ok) { + return { + decision: { + ...baseDecision, + ...buildResult.details, + price_freshness_ms: String(priceAgeMs), + inventory_freshness_ms: String(inventoryAgeMs), + decision_reason: buildResult.reason, + }, + }; + } + + const decision = { + ...baseDecision, + ...buildResult.details, + price_freshness_ms: String(priceAgeMs), + inventory_freshness_ms: String(inventoryAgeMs), + decision: armed ? 'actionable' : 'rejected', + decision_reason: armed ? 'actionable' : 'strategy_disarmed', + }; + + if (!armed) return { decision }; + + return { + decision, + command: { + command_id: `cmd-${decisionId}`, + decision_id: decisionId, + idempotency_key: `quote:${payload.quote_id}`, + execution_key: payload.quote_id, + quote_id: payload.quote_id, + pair: decision.pair, + asset_in: payload.asset_in, + asset_out: payload.asset_out, + amount_in: payload.amount_in ?? null, + amount_out: payload.amount_out ?? null, + request_kind: payload.request_kind, + min_deadline_ms: payload.min_deadline_ms ?? '60000', + quote_output: buildResult.quoteOutput, + proposed_amount_in: buildResult.details.proposed_amount_in ?? null, + proposed_amount_out: buildResult.details.proposed_amount_out ?? null, + }, + }; +} + +function buildQuote({ + demand, + price, + inventory, + config, + thresholdPct, + maxNotionalEure, +}) { + const direction = classifyPairDirection({ + assetIn: demand.asset_in, + assetOut: demand.asset_out, + btcAssetId: config.tradingBtc.assetId, + eureAssetId: config.tradingEure.assetId, + }); + + if (direction === 'unsupported') { + return { ok: false, reason: 'unsupported_pair', details: {} }; + } + + const thresholdFactor = 1 - (thresholdPct / 100); + const penaltyFactor = 1 + (thresholdPct / 100); + const spendAsset = demand.asset_out; + const available = bigintAmount(inventory.spendable?.[spendAsset] || '0'); + const pendingInbound = bigintAmount(inventory.pending_inbound?.[spendAsset] || '0'); + + if (demand.request_kind === 'exact_in') { + const amountIn = bigintAmount(demand.amount_in); + if (amountIn <= 0n) { + return { ok: false, reason: 'invalid_amount', details: {} }; + } + + const inputNumber = unitsToNumber( + demand.amount_in, + config.assetRegistry.get(demand.asset_in).decimals, + ); + const fairOutput = direction === 'btc_to_eure' + ? inputNumber * Number(price.eure_per_btc) + : inputNumber * Number(price.btc_per_eure); + const proposedOutput = fairOutput * thresholdFactor; + const proposedOutputUnits = numberToUnits( + proposedOutput, + config.assetRegistry.get(demand.asset_out).decimals, + { mode: 'floor' }, + ); + const spendRequired = bigintAmount(proposedOutputUnits); + const eureNotional = direction === 'btc_to_eure' + ? fairOutput + : inputNumber; + const impliedRate = unitsToNumber( + proposedOutputUnits, + config.assetRegistry.get(demand.asset_out).decimals, + ) / inputNumber; + const referenceRate = direction === 'btc_to_eure' + ? Number(price.eure_per_btc) + : Number(price.btc_per_eure); + + return finalizeQuote({ + direction, + available, + pendingInbound, + spendAsset, + spendRequired, + eureNotional, + maxNotionalEure, + proposedAmountOut: proposedOutputUnits, + impliedRate, + referenceRate, + inventoryId: inventory.inventory_id, + priceId: price.price_id, + }); + } + + if (demand.request_kind === 'exact_out') { + const amountOut = bigintAmount(demand.amount_out); + if (amountOut <= 0n) { + return { ok: false, reason: 'invalid_amount', details: {} }; + } + + const outputNumber = unitsToNumber( + demand.amount_out, + config.assetRegistry.get(demand.asset_out).decimals, + ); + const fairInput = direction === 'btc_to_eure' + ? outputNumber * Number(price.btc_per_eure) + : outputNumber * Number(price.eure_per_btc); + const proposedInput = fairInput * penaltyFactor; + const proposedInputUnits = numberToUnits( + proposedInput, + config.assetRegistry.get(demand.asset_in).decimals, + { mode: 'ceil' }, + ); + const spendRequired = amountOut; + const eureNotional = direction === 'btc_to_eure' + ? outputNumber + : fairInput; + const impliedRate = outputNumber / unitsToNumber( + proposedInputUnits, + config.assetRegistry.get(demand.asset_in).decimals, + ); + const referenceRate = direction === 'btc_to_eure' + ? Number(price.eure_per_btc) + : Number(price.btc_per_eure); + + return finalizeQuote({ + direction, + available, + pendingInbound, + spendAsset, + spendRequired, + eureNotional, + maxNotionalEure, + proposedAmountIn: proposedInputUnits, + proposedAmountOut: demand.amount_out, + impliedRate, + referenceRate, + inventoryId: inventory.inventory_id, + priceId: price.price_id, + }); + } + + return { ok: false, reason: 'unsupported_request_kind', details: {} }; +} + +function finalizeQuote({ + direction, + available, + pendingInbound, + spendAsset, + spendRequired, + eureNotional, + maxNotionalEure, + proposedAmountIn = null, + proposedAmountOut = null, + impliedRate, + referenceRate, + inventoryId, + priceId, +}) { + const grossEdgePct = ((referenceRate - impliedRate) / referenceRate) * 100; + const reasonBase = { + direction, + reference_rate: formatNumber(referenceRate, 12), + implied_rate: formatNumber(impliedRate, 12), + gross_edge_pct: formatNumber(grossEdgePct, 6), + inventory_asset: spendAsset, + inventory_required: spendRequired.toString(), + inventory_available: available.toString(), + inventory_id: inventoryId, + price_id: priceId, + eure_notional: formatNumber(eureNotional, 6), + proposed_amount_in: proposedAmountIn, + proposed_amount_out: proposedAmountOut, + }; + + if (!(grossEdgePct >= 0)) { + return { ok: false, reason: 'invalid_pricing', details: reasonBase }; + } + + if (eureNotional > maxNotionalEure) { + return { ok: false, reason: 'max_notional_exceeded', details: reasonBase }; + } + + if (spendRequired <= 0n) { + return { ok: false, reason: 'invalid_quote_output', details: reasonBase }; + } + + if (available < spendRequired) { + return { + ok: false, + reason: pendingInbound > 0n ? 'pending_deposit_not_credited' : 'insufficient_inventory', + details: { + ...reasonBase, + pending_inbound: pendingInbound.toString(), + }, + }; + } + + return { + ok: true, + details: reasonBase, + quoteOutput: compact({ + amount_in: proposedAmountIn, + amount_out: proposedAmountOut, + }), + }; +} + +function compact(value) { + return Object.fromEntries( + Object.entries(value).filter(([, entry]) => entry != null), + ); +} + +function withReason(decision, reason) { + return { + ...decision, + decision_reason: reason, + }; +} diff --git a/src/lib/config.mjs b/src/lib/config.mjs index 780426d..9dcca0f 100644 --- a/src/lib/config.mjs +++ b/src/lib/config.mjs @@ -3,22 +3,64 @@ import { DEFAULT_NEAR_INTENTS_PAIR_FILTER } from '../core/pair-filter.mjs'; const DEFAULTS = { nearIntentsWsUrl: 'wss://solver-relay-v2.chaindefuser.com/ws', + nearIntentsRpcUrl: 'https://solver-relay-v2.chaindefuser.com/rpc', + nearBridgeRpcUrl: 'https://bridge.chaindefuser.com/rpc', + nearRpcUrl: 'https://rpc.fastnear.com', + nearVerifierContract: 'intents.near', nearIntentsPairFilter: DEFAULT_NEAR_INTENTS_PAIR_FILTER, nearIntentsPairFilterReloadMs: 5_000, nearIntentsControlApiEnabled: true, nearIntentsControlHost: '0.0.0.0', nearIntentsControlPort: 8081, + marketReferenceControlPort: 8082, + inventorySyncControlPort: 8083, + liquidityManagerControlPort: 8084, + historyWriterControlPort: 8085, + strategyEngineControlPort: 8086, + tradeExecutorControlPort: 8087, kafkaBrokers: ['127.0.0.1:9092'], kafkaClientId: 'unrip', kafkaTopicRawNearIntentsQuote: 'raw.near_intents.quote', kafkaTopicNormSwapDemand: 'norm.swap_demand', + kafkaTopicRefMarketPrice: 'ref.market_price', + kafkaTopicStateIntentInventory: 'state.intent_inventory', + kafkaTopicOpsLiquidityAction: 'ops.liquidity_action', + kafkaTopicDecisionTradeDecision: 'decision.trade_decision', kafkaTopicCmdExecuteTrade: 'cmd.execute_trade', kafkaTopicExecTradeResult: 'exec.trade_result', - kafkaConsumerGroupDummy: 'dummy-reactor-v1', - kafkaConsumerGroupExecutor: 'dummy-executor-v1', + kafkaConsumerGroupHistory: 'history-writer-v1', + kafkaConsumerGroupInventory: 'inventory-sync-v1', + kafkaConsumerGroupStrategy: 'strategy-engine-v1', + kafkaConsumerGroupExecutor: 'trade-executor-v1', executorStateDir: './var/executor-state', + liquidityStateDir: './var/liquidity-state', + postgresUrl: 'postgresql://unrip:unrip@127.0.0.1:5432/unrip', projectName: 'unrip', projectNamespace: 'unrip', + tradingBtcAssetId: 'nep141:btc.omft.near', + tradingBtcSymbol: 'BTC', + tradingBtcDecimals: 8, + tradingBtcChain: 'btc:mainnet', + tradingEureAssetId: 'nep141:gnosis-0x420ca0f9b9b604ce0fd9c18ef134c705e5fa3430.omft.near', + tradingEureSymbol: 'EURe', + tradingEureDecimals: 18, + tradingEureChain: 'eth:100', + marketReferenceRefreshMs: 5_000, + marketReferenceCoinGeckoRefreshMs: 15_000, + marketReferenceMaxAgeMs: 30_000, + marketReferenceKrakenTickerUrl: 'https://api.kraken.com/0/public/Ticker?pair=XBTEUR', + marketReferenceCoinGeckoUrl: + 'https://api.coingecko.com/api/v3/simple/price?ids=bitcoin&vs_currencies=eur', + inventorySyncRefreshMs: 15_000, + liquidityRefreshMs: 30_000, + strategyGrossThresholdPct: 2, + strategyInitialArmed: false, + strategyMaxNotionalEure: 5, + strategyPriceMaxAgeMs: 30_000, + strategyInventoryMaxAgeMs: 30_000, + executorInitialArmed: false, + executorResponseTimeoutMs: 10_000, + withdrawalsFrozen: true, }; function splitCsv(value) { @@ -28,53 +70,6 @@ function splitCsv(value) { .filter(Boolean); } -export function loadConfig({ envPath = '.env' } = {}) { - // Runtime config stays environment-first so the same app build works for: - // - local `.env` development - // - Docker/Compose - // - Kubernetes Secret/ConfigMap injection during Hetzner bootstrap - // This is what lets the local workstation bootstrap provision infra and then - // deploy the exact same image into k3s without app-level config rewrites. - loadDotenv(envPath); - - return { - nearIntentsApiKey: process.env.NEAR_INTENTS_API_KEY || '', - nearIntentsWsUrl: process.env.NEAR_INTENTS_WS_URL || DEFAULTS.nearIntentsWsUrl, - nearIntentsPairFilter: - process.env.NEAR_INTENTS_PAIR_FILTER || DEFAULTS.nearIntentsPairFilter, - nearIntentsPairFilterFile: process.env.NEAR_INTENTS_PAIR_FILTER_FILE || '', - nearIntentsPairFilterReloadMs: - parseNumber(process.env.NEAR_INTENTS_PAIR_FILTER_RELOAD_MS, DEFAULTS.nearIntentsPairFilterReloadMs), - nearIntentsControlApiEnabled: - parseBoolean(process.env.NEAR_INTENTS_CONTROL_API_ENABLED, DEFAULTS.nearIntentsControlApiEnabled), - nearIntentsControlHost: - process.env.NEAR_INTENTS_CONTROL_HOST || DEFAULTS.nearIntentsControlHost, - nearIntentsControlPort: - parseNumber(process.env.NEAR_INTENTS_CONTROL_PORT, DEFAULTS.nearIntentsControlPort), - kafkaBrokers: splitCsv(process.env.KAFKA_BROKERS).length - ? splitCsv(process.env.KAFKA_BROKERS) - : DEFAULTS.kafkaBrokers, - kafkaClientId: process.env.KAFKA_CLIENT_ID || DEFAULTS.kafkaClientId, - kafkaTopicRawNearIntentsQuote: - process.env.KAFKA_TOPIC_RAW_NEAR_INTENTS_QUOTE || DEFAULTS.kafkaTopicRawNearIntentsQuote, - kafkaTopicNormSwapDemand: - process.env.KAFKA_TOPIC_NORM_SWAP_DEMAND || DEFAULTS.kafkaTopicNormSwapDemand, - kafkaTopicCmdExecuteTrade: - process.env.KAFKA_TOPIC_CMD_EXECUTE_TRADE || DEFAULTS.kafkaTopicCmdExecuteTrade, - kafkaTopicExecTradeResult: - process.env.KAFKA_TOPIC_EXEC_TRADE_RESULT || DEFAULTS.kafkaTopicExecTradeResult, - kafkaConsumerGroupDummy: - process.env.KAFKA_CONSUMER_GROUP_DUMMY || DEFAULTS.kafkaConsumerGroupDummy, - kafkaConsumerGroupExecutor: - process.env.KAFKA_CONSUMER_GROUP_EXECUTOR || DEFAULTS.kafkaConsumerGroupExecutor, - executorStateDir: - process.env.EXECUTOR_STATE_DIR || DEFAULTS.executorStateDir, - projectName: process.env.PROJECT_NAME || DEFAULTS.projectName, - projectNamespace: - process.env.PROJECT_NAMESPACE || process.env.PROJECT_NAME || DEFAULTS.projectNamespace, - }; -} - function parseNumber(value, fallback) { const parsed = Number(value); return Number.isFinite(parsed) ? parsed : fallback; @@ -88,3 +83,195 @@ function parseBoolean(value, fallback) { if (['0', 'false', 'no', 'off'].includes(normalized)) return false; return fallback; } + +function buildAsset({ assetId, symbol, decimals, chain }) { + return { + assetId, + symbol, + decimals, + chain, + }; +} + +export function loadConfig({ envPath = '.env' } = {}) { + loadDotenv(envPath); + + const tradingBtc = buildAsset({ + assetId: process.env.TRADING_BTC_ASSET_ID || DEFAULTS.tradingBtcAssetId, + symbol: process.env.TRADING_BTC_SYMBOL || DEFAULTS.tradingBtcSymbol, + decimals: parseNumber(process.env.TRADING_BTC_DECIMALS, DEFAULTS.tradingBtcDecimals), + chain: process.env.TRADING_BTC_CHAIN || DEFAULTS.tradingBtcChain, + }); + const tradingEure = buildAsset({ + assetId: process.env.TRADING_EURE_ASSET_ID || DEFAULTS.tradingEureAssetId, + symbol: process.env.TRADING_EURE_SYMBOL || DEFAULTS.tradingEureSymbol, + decimals: parseNumber(process.env.TRADING_EURE_DECIMALS, DEFAULTS.tradingEureDecimals), + chain: process.env.TRADING_EURE_CHAIN || DEFAULTS.tradingEureChain, + }); + + const projectName = process.env.PROJECT_NAME || DEFAULTS.projectName; + const projectNamespace = + process.env.PROJECT_NAMESPACE || projectName || DEFAULTS.projectNamespace; + + return { + nearIntentsApiKey: process.env.NEAR_INTENTS_API_KEY || '', + nearIntentsAccountId: process.env.NEAR_INTENTS_ACCOUNT_ID || '', + nearIntentsSignerPrivateKey: process.env.NEAR_INTENTS_SIGNER_PRIVATE_KEY || '', + nearIntentsWsUrl: process.env.NEAR_INTENTS_WS_URL || DEFAULTS.nearIntentsWsUrl, + nearIntentsRpcUrl: process.env.NEAR_INTENTS_RPC_URL || DEFAULTS.nearIntentsRpcUrl, + nearBridgeRpcUrl: process.env.NEAR_INTENTS_BRIDGE_RPC_URL || DEFAULTS.nearBridgeRpcUrl, + nearRpcUrl: process.env.NEAR_RPC_URL || DEFAULTS.nearRpcUrl, + nearVerifierContract: + process.env.NEAR_INTENTS_VERIFIER_CONTRACT || DEFAULTS.nearVerifierContract, + nearIntentsPairFilter: + process.env.NEAR_INTENTS_PAIR_FILTER || DEFAULTS.nearIntentsPairFilter, + nearIntentsPairFilterFile: process.env.NEAR_INTENTS_PAIR_FILTER_FILE || '', + nearIntentsPairFilterReloadMs: parseNumber( + process.env.NEAR_INTENTS_PAIR_FILTER_RELOAD_MS, + DEFAULTS.nearIntentsPairFilterReloadMs, + ), + nearIntentsControlApiEnabled: parseBoolean( + process.env.NEAR_INTENTS_CONTROL_API_ENABLED, + DEFAULTS.nearIntentsControlApiEnabled, + ), + nearIntentsControlHost: + process.env.NEAR_INTENTS_CONTROL_HOST || DEFAULTS.nearIntentsControlHost, + nearIntentsControlPort: parseNumber( + process.env.NEAR_INTENTS_CONTROL_PORT, + DEFAULTS.nearIntentsControlPort, + ), + marketReferenceControlHost: + process.env.MARKET_REFERENCE_CONTROL_HOST || DEFAULTS.nearIntentsControlHost, + marketReferenceControlPort: parseNumber( + process.env.MARKET_REFERENCE_CONTROL_PORT, + DEFAULTS.marketReferenceControlPort, + ), + inventorySyncControlHost: + process.env.INVENTORY_SYNC_CONTROL_HOST || DEFAULTS.nearIntentsControlHost, + inventorySyncControlPort: parseNumber( + process.env.INVENTORY_SYNC_CONTROL_PORT, + DEFAULTS.inventorySyncControlPort, + ), + liquidityManagerControlHost: + process.env.LIQUIDITY_MANAGER_CONTROL_HOST || DEFAULTS.nearIntentsControlHost, + liquidityManagerControlPort: parseNumber( + process.env.LIQUIDITY_MANAGER_CONTROL_PORT, + DEFAULTS.liquidityManagerControlPort, + ), + historyWriterControlHost: + process.env.HISTORY_WRITER_CONTROL_HOST || DEFAULTS.nearIntentsControlHost, + historyWriterControlPort: parseNumber( + process.env.HISTORY_WRITER_CONTROL_PORT, + DEFAULTS.historyWriterControlPort, + ), + strategyEngineControlHost: + process.env.STRATEGY_ENGINE_CONTROL_HOST || DEFAULTS.nearIntentsControlHost, + strategyEngineControlPort: parseNumber( + process.env.STRATEGY_ENGINE_CONTROL_PORT, + DEFAULTS.strategyEngineControlPort, + ), + tradeExecutorControlHost: + process.env.TRADE_EXECUTOR_CONTROL_HOST || DEFAULTS.nearIntentsControlHost, + tradeExecutorControlPort: parseNumber( + process.env.TRADE_EXECUTOR_CONTROL_PORT, + DEFAULTS.tradeExecutorControlPort, + ), + kafkaBrokers: splitCsv(process.env.KAFKA_BROKERS).length + ? splitCsv(process.env.KAFKA_BROKERS) + : DEFAULTS.kafkaBrokers, + kafkaClientId: process.env.KAFKA_CLIENT_ID || DEFAULTS.kafkaClientId, + kafkaTopicRawNearIntentsQuote: + process.env.KAFKA_TOPIC_RAW_NEAR_INTENTS_QUOTE || DEFAULTS.kafkaTopicRawNearIntentsQuote, + kafkaTopicNormSwapDemand: + process.env.KAFKA_TOPIC_NORM_SWAP_DEMAND || DEFAULTS.kafkaTopicNormSwapDemand, + kafkaTopicRefMarketPrice: + process.env.KAFKA_TOPIC_REF_MARKET_PRICE || DEFAULTS.kafkaTopicRefMarketPrice, + kafkaTopicStateIntentInventory: + process.env.KAFKA_TOPIC_STATE_INTENT_INVENTORY || DEFAULTS.kafkaTopicStateIntentInventory, + kafkaTopicOpsLiquidityAction: + process.env.KAFKA_TOPIC_OPS_LIQUIDITY_ACTION || DEFAULTS.kafkaTopicOpsLiquidityAction, + kafkaTopicDecisionTradeDecision: + process.env.KAFKA_TOPIC_DECISION_TRADE_DECISION || DEFAULTS.kafkaTopicDecisionTradeDecision, + kafkaTopicCmdExecuteTrade: + process.env.KAFKA_TOPIC_CMD_EXECUTE_TRADE || DEFAULTS.kafkaTopicCmdExecuteTrade, + kafkaTopicExecTradeResult: + process.env.KAFKA_TOPIC_EXEC_TRADE_RESULT || DEFAULTS.kafkaTopicExecTradeResult, + kafkaConsumerGroupHistory: + process.env.KAFKA_CONSUMER_GROUP_HISTORY || DEFAULTS.kafkaConsumerGroupHistory, + kafkaConsumerGroupInventory: + process.env.KAFKA_CONSUMER_GROUP_INVENTORY || DEFAULTS.kafkaConsumerGroupInventory, + kafkaConsumerGroupStrategy: + process.env.KAFKA_CONSUMER_GROUP_STRATEGY || DEFAULTS.kafkaConsumerGroupStrategy, + kafkaConsumerGroupExecutor: + process.env.KAFKA_CONSUMER_GROUP_EXECUTOR || DEFAULTS.kafkaConsumerGroupExecutor, + executorStateDir: process.env.EXECUTOR_STATE_DIR || DEFAULTS.executorStateDir, + liquidityStateDir: process.env.LIQUIDITY_STATE_DIR || DEFAULTS.liquidityStateDir, + postgresUrl: process.env.POSTGRES_URL || DEFAULTS.postgresUrl, + projectName, + projectNamespace, + tradingBtc, + tradingEure, + activePair: `${tradingBtc.assetId}->${tradingEure.assetId}`, + activeAssetIds: [tradingBtc.assetId, tradingEure.assetId], + assetRegistry: new Map([ + [tradingBtc.assetId, tradingBtc], + [tradingEure.assetId, tradingEure], + ]), + marketReferenceRefreshMs: parseNumber( + process.env.MARKET_REFERENCE_REFRESH_MS, + DEFAULTS.marketReferenceRefreshMs, + ), + marketReferenceCoinGeckoRefreshMs: parseNumber( + process.env.MARKET_REFERENCE_COINGECKO_REFRESH_MS, + DEFAULTS.marketReferenceCoinGeckoRefreshMs, + ), + marketReferenceMaxAgeMs: parseNumber( + process.env.MARKET_REFERENCE_MAX_AGE_MS, + DEFAULTS.marketReferenceMaxAgeMs, + ), + marketReferenceKrakenTickerUrl: + process.env.MARKET_REFERENCE_KRAKEN_TICKER_URL || DEFAULTS.marketReferenceKrakenTickerUrl, + marketReferenceCoinGeckoUrl: + process.env.MARKET_REFERENCE_COINGECKO_URL || DEFAULTS.marketReferenceCoinGeckoUrl, + inventorySyncRefreshMs: parseNumber( + process.env.INVENTORY_SYNC_REFRESH_MS, + DEFAULTS.inventorySyncRefreshMs, + ), + liquidityRefreshMs: parseNumber( + process.env.LIQUIDITY_REFRESH_MS, + DEFAULTS.liquidityRefreshMs, + ), + strategyGrossThresholdPct: parseNumber( + process.env.STRATEGY_GROSS_THRESHOLD_PCT, + DEFAULTS.strategyGrossThresholdPct, + ), + strategyInitialArmed: parseBoolean( + process.env.STRATEGY_INITIAL_ARMED, + DEFAULTS.strategyInitialArmed, + ), + strategyMaxNotionalEure: parseNumber( + process.env.STRATEGY_MAX_NOTIONAL_EURE, + DEFAULTS.strategyMaxNotionalEure, + ), + strategyPriceMaxAgeMs: parseNumber( + process.env.STRATEGY_PRICE_MAX_AGE_MS, + DEFAULTS.strategyPriceMaxAgeMs, + ), + strategyInventoryMaxAgeMs: parseNumber( + process.env.STRATEGY_INVENTORY_MAX_AGE_MS, + DEFAULTS.strategyInventoryMaxAgeMs, + ), + executorInitialArmed: parseBoolean( + process.env.EXECUTOR_INITIAL_ARMED, + DEFAULTS.executorInitialArmed, + ), + executorResponseTimeoutMs: parseNumber( + process.env.EXECUTOR_RESPONSE_TIMEOUT_MS, + DEFAULTS.executorResponseTimeoutMs, + ), + withdrawalsFrozen: parseBoolean( + process.env.LIQUIDITY_WITHDRAWALS_FROZEN, + DEFAULTS.withdrawalsFrozen, + ), + }; +} diff --git a/src/lib/http.mjs b/src/lib/http.mjs new file mode 100644 index 0000000..9bf281b --- /dev/null +++ b/src/lib/http.mjs @@ -0,0 +1,22 @@ +export async function fetchJson(url, options = {}) { + const response = await fetch(url, options); + const text = await response.text(); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${text.slice(0, 200)}`); + } + + return text ? JSON.parse(text) : null; +} + +export async function postJson(url, body, { headers = {}, ...options } = {}) { + return fetchJson(url, { + ...options, + method: 'POST', + headers: { + 'content-type': 'application/json', + ...headers, + }, + body: JSON.stringify(body), + }); +} diff --git a/src/lib/market-data.mjs b/src/lib/market-data.mjs new file mode 100644 index 0000000..d15bba4 --- /dev/null +++ b/src/lib/market-data.mjs @@ -0,0 +1,20 @@ +import { fetchJson } from './http.mjs'; + +export async function fetchKrakenBtcEur(url) { + const response = await fetchJson(url); + const pair = Object.values(response.result || {})[0]; + const lastTrade = pair?.c?.[0] || pair?.a?.[0]; + if (!lastTrade) throw new Error('Kraken price missing'); + return Number(lastTrade); +} + +export async function fetchCoinGeckoBtcEur(url) { + const response = await fetchJson(url, { + headers: { + accept: 'application/json', + }, + }); + const price = response?.bitcoin?.eur; + if (!Number.isFinite(price)) throw new Error('CoinGecko price missing'); + return Number(price); +} diff --git a/src/lib/postgres.mjs b/src/lib/postgres.mjs new file mode 100644 index 0000000..c4e34d7 --- /dev/null +++ b/src/lib/postgres.mjs @@ -0,0 +1,89 @@ +import { Pool } from 'pg'; + +const TABLES = [ + 'raw_near_intents_quotes', + 'swap_demand_events', + 'market_price_events', + 'intent_inventory_snapshots', + 'liquidity_actions', + 'trade_decisions', + 'execute_trade_commands', + 'trade_execution_results', +]; + +export function createPostgresPool({ connectionString }) { + return new Pool({ + connectionString, + }); +} + +export async function ensureHistorySchema(pool) { + for (const table of TABLES) { + await pool.query(` + CREATE TABLE IF NOT EXISTS ${table} ( + event_id TEXT PRIMARY KEY, + topic TEXT NOT NULL, + venue TEXT NOT NULL, + source TEXT, + event_type TEXT NOT NULL, + observed_at TIMESTAMPTZ, + ingested_at TIMESTAMPTZ NOT NULL, + quote_id TEXT, + pair TEXT, + decision_key TEXT, + payload JSONB NOT NULL, + raw JSONB + ) + `); + await pool.query(` + CREATE INDEX IF NOT EXISTS ${table}_quote_id_idx + ON ${table} (quote_id) + `); + await pool.query(` + CREATE INDEX IF NOT EXISTS ${table}_decision_key_idx + ON ${table} (decision_key) + `); + await pool.query(` + CREATE INDEX IF NOT EXISTS ${table}_ingested_at_idx + ON ${table} (ingested_at DESC) + `); + } +} + +export async function insertHistoryEvent(pool, { table, topic, event, record }) { + await pool.query( + ` + INSERT INTO ${table} ( + event_id, + topic, + venue, + source, + event_type, + observed_at, + ingested_at, + quote_id, + pair, + decision_key, + payload, + raw + ) VALUES ( + $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12::jsonb + ) + ON CONFLICT (event_id) DO NOTHING + `, + [ + event.event_id, + topic, + event.venue, + event.source, + event.event_type, + event.observed_at, + event.ingested_at, + record.quote_id, + record.pair, + record.decision_key, + JSON.stringify(event.payload), + event.raw ? JSON.stringify(event.raw) : null, + ], + ); +} diff --git a/src/venues/near-intents/bridge-client.mjs b/src/venues/near-intents/bridge-client.mjs new file mode 100644 index 0000000..46f33c6 --- /dev/null +++ b/src/venues/near-intents/bridge-client.mjs @@ -0,0 +1,49 @@ +import { postJson } from '../../lib/http.mjs'; + +export function createNearBridgeClient({ rpcUrl }) { + let id = 1; + + async function rpc(method, params) { + const response = await postJson(rpcUrl, { + jsonrpc: '2.0', + id: id++, + method, + params: [params], + }); + + if (response.error) { + throw new Error(response.error.message || `Bridge RPC ${method} failed`); + } + + return response.result; + } + + return { + supportedTokens({ chains }) { + return rpc('supported_tokens', { chains }); + }, + depositAddress({ accountId, chain }) { + return rpc('deposit_address', { + account_id: accountId, + chain, + }); + }, + recentDeposits({ accountId, chain }) { + return rpc('recent_deposits', { + account_id: accountId, + chain, + }); + }, + withdrawalStatus({ withdrawalHash }) { + return rpc('withdrawal_status', { + withdrawal_hash: withdrawalHash, + }); + }, + notifyDeposit({ depositAddress, txHash }) { + return rpc('notify_deposit', { + deposit_address: depositAddress, + tx_hash: txHash, + }); + }, + }; +} diff --git a/src/venues/near-intents/normalize.mjs b/src/venues/near-intents/normalize.mjs index 8bd60cc..cd29cb6 100644 --- a/src/venues/near-intents/normalize.mjs +++ b/src/venues/near-intents/normalize.mjs @@ -42,13 +42,20 @@ export function normalizeNearIntentsQuote(message) { const assetOut = first(message, ['defuse_asset_identifier_out', 'buyToken', 'asset_out']); if (!quoteId || !assetIn || !assetOut) return null; + const amountIn = stringify(first(message, ['exact_amount_in', 'sellAmount', 'amount_in'])); + const amountOut = stringify(first(message, ['exact_amount_out', 'buyAmount', 'amount_out', 'expectedOut', 'quoted_amount_out'])); + const requestKind = amountIn != null ? 'exact_in' : amountOut != null ? 'exact_out' : null; + if (!requestKind) return null; + return { quote_id: String(quoteId), asset_in: String(assetIn), asset_out: String(assetOut), - amount_in: stringify(first(message, ['exact_amount_in', 'sellAmount', 'amount_in'])), - amount_out: stringify(first(message, ['exact_amount_out', 'buyAmount', 'amount_out', 'expectedOut', 'quoted_amount_out'])), - ttl_ms: stringify(first(message, ['min_deadline_ms', 'ttl_ms', 'deadline_ms'])), + pair: `${String(assetIn)}->${String(assetOut)}`, + request_kind: requestKind, + amount_in: amountIn, + amount_out: amountOut, + min_deadline_ms: stringify(first(message, ['min_deadline_ms', 'ttl_ms', 'deadline_ms'])), }; } diff --git a/src/venues/near-intents/signing.mjs b/src/venues/near-intents/signing.mjs new file mode 100644 index 0000000..671d287 --- /dev/null +++ b/src/venues/near-intents/signing.mjs @@ -0,0 +1,86 @@ +import crypto from 'node:crypto'; + +export function buildIntentNonce(currentSaltHex) { + const salt = Buffer.from(String(currentSaltHex || '').replace(/^0x/, ''), 'hex'); + if (salt.length !== 4) throw new Error('current_salt must be 4 bytes in hex'); + return Buffer.concat([salt, crypto.randomBytes(28)]).toString('base64'); +} + +export function buildQuoteResponseSubmission({ + command, + signerAccountId, + signer, + verifierContract, + currentSaltHex, + now = Date.now(), +}) { + const deadline = new Date(now + Number(command.min_deadline_ms || 60_000)).toISOString(); + const nonce = buildIntentNonce(currentSaltHex); + const receiveAmount = command.request_kind === 'exact_in' + ? command.amount_in + : command.quote_output.amount_in; + const sendAmount = command.request_kind === 'exact_in' + ? command.quote_output.amount_out + : command.amount_out; + + const payload = { + signer_id: signerAccountId, + verifying_contract: verifierContract, + deadline, + nonce, + intents: [ + { + intent: 'token_diff', + diff: { + [command.asset_in]: String(receiveAmount), + [command.asset_out]: `-${String(sendAmount)}`, + }, + }, + ], + }; + + const payloadString = JSON.stringify(payload); + const signed = signer.sign(Buffer.from(payloadString)); + + return { + quote_id: command.quote_id, + quote_output: command.quote_output, + signed_data: { + standard: 'raw_ed25519', + payload: payloadString, + public_key: signer.getPublicKey().toString(), + signature: encodeNearSignature(signed.signature), + }, + }; +} + +function encodeNearSignature(signatureBytes) { + return `ed25519:${encodeBase58(signatureBytes)}`; +} + +function encodeBase58(bytes) { + const alphabet = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'; + if (!bytes || bytes.length === 0) return ''; + + const digits = [0]; + for (const byte of bytes) { + let carry = byte; + for (let index = 0; index < digits.length; index += 1) { + carry += digits[index] * 256; + digits[index] = carry % 58; + carry = Math.floor(carry / 58); + } + while (carry > 0) { + digits.push(carry % 58); + carry = Math.floor(carry / 58); + } + } + + let prefix = ''; + for (const byte of bytes) { + if (byte === 0) prefix += alphabet[0]; + else break; + } + + return prefix + digits.reverse().map((digit) => alphabet[digit]).join(''); +} diff --git a/src/venues/near-intents/solver-relay-ws.mjs b/src/venues/near-intents/solver-relay-ws.mjs new file mode 100644 index 0000000..0da4c2a --- /dev/null +++ b/src/venues/near-intents/solver-relay-ws.mjs @@ -0,0 +1,156 @@ +import { serializeError } from '../../core/log.mjs'; + +export async function startSolverRelayWs({ + apiKey, + wsUrl, + logger = null, + subscriptions = [], + onEvent = () => {}, +}) { + if (!apiKey) throw new Error('Missing NEAR_INTENTS_API_KEY'); + + let socket = null; + let closed = false; + let reconnectTimer = null; + let connected = false; + let requestId = 1; + const pending = new Map(); + let readyResolvers = []; + + connect(); + + return { + async request(method, params, { timeoutMs = 10_000 } = {}) { + await waitForConnection(); + + const id = requestId++; + const payload = { + jsonrpc: '2.0', + id, + method, + params, + }; + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + pending.delete(id); + reject(new Error(`${method} timed out`)); + }, timeoutMs); + + pending.set(id, { + resolve(result) { + clearTimeout(timeout); + resolve(result); + }, + reject(error) { + clearTimeout(timeout); + reject(error); + }, + }); + + socket.send(JSON.stringify(payload)); + }); + }, + getState() { + return { + connected, + pending_requests: pending.size, + }; + }, + close() { + closed = true; + if (reconnectTimer) clearTimeout(reconnectTimer); + if (socket && socket.readyState <= 1) socket.close(); + rejectAllPending(new Error('Socket closed')); + }, + }; + + function connect() { + if (closed) return; + + socket = new WebSocket(wsUrl, { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + }); + + socket.addEventListener('open', () => { + connected = true; + logger?.info('connection_established', { + venue: 'near-intents', + }); + resolveReady(); + + for (const subscription of subscriptions) { + socket.send(JSON.stringify({ + jsonrpc: '2.0', + id: requestId++, + method: 'subscribe', + params: [subscription], + })); + } + }); + + socket.addEventListener('message', (event) => { + const text = typeof event.data === 'string' ? event.data : Buffer.from(event.data).toString('utf8'); + let payload; + try { + payload = JSON.parse(text); + } catch { + logger?.warn('invalid_json_message', { + venue: 'near-intents', + }); + return; + } + + if (payload.id && pending.has(payload.id)) { + const request = pending.get(payload.id); + pending.delete(payload.id); + if (payload.error) request.reject(new Error(payload.error.message || 'RPC failed')); + else request.resolve(payload.result); + return; + } + + onEvent(payload); + }); + + socket.addEventListener('close', () => { + connected = false; + rejectAllPending(new Error('Socket disconnected')); + logger?.warn('connection_lost', { + venue: 'near-intents', + details: { + reconnect_in_ms: 2000, + }, + }); + + if (!closed) reconnectTimer = setTimeout(connect, 2000); + }); + + socket.addEventListener('error', (error) => { + logger?.error('socket_error', { + venue: 'near-intents', + details: { + error: serializeError(error), + }, + }); + }); + } + + function waitForConnection() { + if (connected) return Promise.resolve(); + return new Promise((resolve) => { + readyResolvers.push(resolve); + }); + } + + function resolveReady() { + for (const resolve of readyResolvers) resolve(); + readyResolvers = []; + } + + function rejectAllPending(error) { + for (const request of pending.values()) request.reject(error); + pending.clear(); + } +} diff --git a/src/venues/near-intents/verifier-client.mjs b/src/venues/near-intents/verifier-client.mjs new file mode 100644 index 0000000..5393c4f --- /dev/null +++ b/src/venues/near-intents/verifier-client.mjs @@ -0,0 +1,78 @@ +import { JsonRpcProvider, KeyPair } from 'near-api-js'; + +import { postJson } from '../../lib/http.mjs'; + +export function createVerifierClient({ + nearRpcUrl, + verifierContract, + signerPrivateKey = '', +}) { + const provider = new JsonRpcProvider({ url: nearRpcUrl }); + const signer = signerPrivateKey ? KeyPair.fromString(signerPrivateKey) : null; + + return { + async currentSalt() { + return callView(provider, verifierContract, 'current_salt', {}); + }, + async mtBatchBalanceOf({ accountId, tokenIds }) { + const result = await callView(provider, verifierContract, 'mt_batch_balance_of', { + account_id: accountId, + token_ids: tokenIds, + }); + + if (!Array.isArray(result)) { + throw new Error('Unexpected mt_batch_balance_of response'); + } + + return Object.fromEntries(tokenIds.map((tokenId, index) => [tokenId, String(result[index] || '0')])); + }, + async isPublicKeyRegistered({ accountId }) { + if (!signer) return false; + const publicKey = signer.getPublicKey().toString(); + const result = await callView(provider, verifierContract, 'has_public_key', { + account_id: accountId, + public_key: publicKey, + }).catch(() => null); + + if (typeof result === 'boolean') return result; + return null; + }, + getSigner() { + return signer; + }, + }; +} + +export function createSolverRelayRpcClient({ rpcUrl }) { + let id = 1; + return { + async getStatus(intentHash) { + const response = await postJson(rpcUrl, { + jsonrpc: '2.0', + id: id++, + method: 'get_status', + params: [{ intent_hash: intentHash }], + }); + + if (response.error) { + throw new Error(response.error.message || 'Solver Relay get_status failed'); + } + + return response.result; + }, + }; +} + +async function callView(provider, accountId, methodName, args) { + const response = await provider.query({ + request_type: 'call_function', + finality: 'final', + account_id: accountId, + method_name: methodName, + args_base64: Buffer.from(JSON.stringify(args)).toString('base64'), + }); + + const bytes = response.result || []; + const text = Buffer.from(bytes).toString('utf8'); + return text ? JSON.parse(text) : null; +} diff --git a/test/executor-state-store.test.mjs b/test/executor-state-store.test.mjs new file mode 100644 index 0000000..0824233 --- /dev/null +++ b/test/executor-state-store.test.mjs @@ -0,0 +1,22 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { createExecutorStateStore } from '../src/core/executor-state-store.mjs'; + +test('executor state store persists processing and completion state', () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'unrip-executor-')); + const store = createExecutorStateStore({ stateDir }); + + store.markProcessing('cmd-1', { quote_id: 'quote-1' }); + assert.equal(store.get('cmd-1').status, 'processing'); + + store.markCompleted('cmd-1', { result_event_id: 'result-1' }); + assert.equal(store.get('cmd-1').status, 'completed'); + + const reloaded = createExecutorStateStore({ stateDir }); + assert.equal(reloaded.get('cmd-1').status, 'completed'); + assert.equal(reloaded.get('cmd-1').result_event_id, 'result-1'); +}); diff --git a/test/inventory-and-history.test.mjs b/test/inventory-and-history.test.mjs new file mode 100644 index 0000000..0c3000e --- /dev/null +++ b/test/inventory-and-history.test.mjs @@ -0,0 +1,64 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { buildInventorySnapshot } from '../src/core/inventory.mjs'; +import { routeHistoryRecord } from '../src/core/history-records.mjs'; + +test('inventory snapshot keeps pending funding out of spendable balances', () => { + const snapshot = buildInventorySnapshot({ + accountId: 'solver.near', + balances: { + 'nep141:btc.omft.near': '1000', + 'nep141:eure.omft.near': '2000', + }, + recentDeposits: [ + { + asset_id: 'nep141:eure.omft.near', + amount: '300', + status: 'PENDING', + }, + ], + trackedWithdrawals: [ + { + withdrawal_hash: 'wd-1', + asset_id: 'nep141:btc.omft.near', + amount: '25', + status: 'PENDING', + }, + ], + assetRegistry: new Map([ + ['nep141:btc.omft.near', { decimals: 8 }], + ['nep141:eure.omft.near', { decimals: 18 }], + ]), + observedAt: '2026-04-02T10:00:00.000Z', + }); + + assert.equal(snapshot.spendable['nep141:eure.omft.near'], '2000'); + assert.equal(snapshot.pending_inbound['nep141:eure.omft.near'], '300'); + assert.equal(snapshot.pending_outbound['nep141:btc.omft.near'], '25'); +}); + +test('history writer routes decision events into the decision table family', () => { + const routed = routeHistoryRecord({ + topic: 'decision.trade_decision', + event: { + event_id: 'evt-1', + event_type: 'trade_decision', + venue: 'near-intents', + schema_version: 1, + ingested_at: '2026-04-02T10:00:00.000Z', + payload: { + decision_id: 'decision-1', + quote_id: 'quote-1', + pair: 'a->b', + direction: 'btc_to_eure', + decision: 'rejected', + decision_reason: 'strategy_disarmed', + }, + }, + }); + + assert.equal(routed.table, 'trade_decisions'); + assert.equal(routed.record.decision_key, 'decision-1'); + assert.equal(routed.record.quote_id, 'quote-1'); +}); diff --git a/test/signing.test.mjs b/test/signing.test.mjs new file mode 100644 index 0000000..b7ddd15 --- /dev/null +++ b/test/signing.test.mjs @@ -0,0 +1,44 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { KeyPair } from 'near-api-js'; + +import { buildIntentNonce, buildQuoteResponseSubmission } from '../src/venues/near-intents/signing.mjs'; + +test('intent nonce uses verifier salt prefix and 32 byte base64 payload', () => { + const nonce = buildIntentNonce('252812b3'); + const decoded = Buffer.from(nonce, 'base64'); + + assert.equal(decoded.length, 32); + assert.equal(decoded.subarray(0, 4).toString('hex'), '252812b3'); +}); + +test('quote response signing builds token_diff payload for solver submission', () => { + const signer = KeyPair.fromRandom('ed25519'); + const submission = buildQuoteResponseSubmission({ + command: { + quote_id: 'quote-1', + asset_in: 'nep141:btc.omft.near', + asset_out: 'nep141:eure.omft.near', + request_kind: 'exact_in', + amount_in: '5000', + amount_out: null, + min_deadline_ms: '60000', + quote_output: { + amount_out: '4900000000000000000', + }, + }, + signerAccountId: 'solver.near', + signer, + verifierContract: 'intents.near', + currentSaltHex: '252812b3', + now: Date.parse('2026-04-02T10:00:00.000Z'), + }); + + assert.equal(submission.quote_id, 'quote-1'); + assert.equal(submission.signed_data.standard, 'raw_ed25519'); + const payload = JSON.parse(submission.signed_data.payload); + assert.equal(payload.signer_id, 'solver.near'); + assert.equal(payload.intents[0].diff['nep141:btc.omft.near'], '5000'); + assert.equal(payload.intents[0].diff['nep141:eure.omft.near'], '-4900000000000000000'); +}); diff --git a/test/strategy.test.mjs b/test/strategy.test.mjs new file mode 100644 index 0000000..8e02e4d --- /dev/null +++ b/test/strategy.test.mjs @@ -0,0 +1,153 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { evaluateTradeOpportunity } from '../src/core/strategy.mjs'; + +function makeConfig() { + const tradingBtc = { + assetId: 'nep141:btc.omft.near', + symbol: 'BTC', + decimals: 8, + chain: 'btc:mainnet', + }; + const tradingEure = { + assetId: 'nep141:eure.omft.near', + symbol: 'EURe', + decimals: 18, + chain: 'eth:100', + }; + + return { + tradingBtc, + tradingEure, + activePair: `${tradingBtc.assetId}->${tradingEure.assetId}`, + assetRegistry: new Map([ + [tradingBtc.assetId, tradingBtc], + [tradingEure.assetId, tradingEure], + ]), + strategyGrossThresholdPct: 2, + strategyMaxNotionalEure: 5, + strategyPriceMaxAgeMs: 30_000, + strategyInventoryMaxAgeMs: 30_000, + }; +} + +function makePriceEvent(overrides = {}) { + return { + ingested_at: new Date('2026-04-02T10:00:00.000Z').toISOString(), + payload: { + price_id: 'price-1', + pair: 'nep141:btc.omft.near->nep141:eure.omft.near', + eur_per_btc: '100000.00000000', + eure_per_btc: '100000.00000000', + btc_per_eur: '0.000010000000', + btc_per_eure: '0.000010000000', + ...overrides, + }, + }; +} + +function makeInventoryEvent(overrides = {}) { + return { + ingested_at: new Date('2026-04-02T10:00:00.000Z').toISOString(), + payload: { + inventory_id: 'inventory-1', + spendable: { + 'nep141:btc.omft.near': '1000000', + 'nep141:eure.omft.near': '10000000000000000000', + }, + pending_inbound: { + 'nep141:btc.omft.near': '0', + 'nep141:eure.omft.near': '0', + }, + ...overrides, + }, + }; +} + +test('strategy emits actionable exact-in BTC -> EURe command when armed and inventory is fresh', () => { + const config = makeConfig(); + const result = evaluateTradeOpportunity({ + demandEvent: { + payload: { + quote_id: 'quote-1', + pair: config.activePair, + asset_in: config.tradingBtc.assetId, + asset_out: config.tradingEure.assetId, + request_kind: 'exact_in', + amount_in: '5000', + min_deadline_ms: '60000', + }, + }, + priceEvent: makePriceEvent(), + inventoryEvent: makeInventoryEvent(), + config, + armed: true, + now: Date.parse('2026-04-02T10:00:05.000Z'), + }); + + assert.equal(result.decision.decision, 'actionable'); + assert.equal(result.decision.decision_reason, 'actionable'); + assert.ok(result.command); + assert.equal(result.command.quote_output.amount_out, '4900000000000000000'); +}); + +test('strategy blocks when pending deposit exists but credited inventory is insufficient', () => { + const config = makeConfig(); + const result = evaluateTradeOpportunity({ + demandEvent: { + payload: { + quote_id: 'quote-2', + pair: config.activePair, + asset_in: config.tradingBtc.assetId, + asset_out: config.tradingEure.assetId, + request_kind: 'exact_in', + amount_in: '5000', + min_deadline_ms: '60000', + }, + }, + priceEvent: makePriceEvent(), + inventoryEvent: makeInventoryEvent({ + spendable: { + 'nep141:btc.omft.near': '1000000', + 'nep141:eure.omft.near': '1000000000000000000', + }, + pending_inbound: { + 'nep141:btc.omft.near': '0', + 'nep141:eure.omft.near': '5000000000000000000', + }, + }), + config, + armed: true, + now: Date.parse('2026-04-02T10:00:05.000Z'), + }); + + assert.equal(result.decision.decision, 'rejected'); + assert.equal(result.decision.decision_reason, 'pending_deposit_not_credited'); + assert.equal(result.command, undefined); +}); + +test('strategy blocks stale prices before command emission', () => { + const config = makeConfig(); + const result = evaluateTradeOpportunity({ + demandEvent: { + payload: { + quote_id: 'quote-3', + pair: config.activePair, + asset_in: config.tradingEure.assetId, + asset_out: config.tradingBtc.assetId, + request_kind: 'exact_out', + amount_out: '10000', + min_deadline_ms: '60000', + }, + }, + priceEvent: makePriceEvent(), + inventoryEvent: makeInventoryEvent(), + config, + armed: true, + now: Date.parse('2026-04-02T10:00:45.000Z'), + }); + + assert.equal(result.decision.decision_reason, 'stale_reference_price'); + assert.equal(result.command, undefined); +});