refactor: isolate unrip project into projects folder
This commit is contained in:
Philipp
parent
15ec32bece
commit
e1869ca93a
51 changed files with 1018 additions and 984 deletions
|
|
@ -15,6 +15,7 @@ jobs:
|
||||||
PROJECT_NAME: ${{ vars.PROJECT_NAME || 'unrip' }}
|
PROJECT_NAME: ${{ vars.PROJECT_NAME || 'unrip' }}
|
||||||
PROJECT_NAMESPACE: ${{ vars.PROJECT_NAMESPACE || vars.PROJECT_NAME || 'unrip' }}
|
PROJECT_NAMESPACE: ${{ vars.PROJECT_NAMESPACE || vars.PROJECT_NAME || 'unrip' }}
|
||||||
PROJECT_DEPLOYMENTS: ${{ vars.PROJECT_DEPLOYMENTS || 'near-intents-ingest,dummy-reactor,dummy-executor,dummy-consumer' }}
|
PROJECT_DEPLOYMENTS: ${{ vars.PROJECT_DEPLOYMENTS || 'near-intents-ingest,dummy-reactor,dummy-executor,dummy-consumer' }}
|
||||||
|
PROJECT_PATH: ${{ vars.PROJECT_PATH || format('projects/{0}', vars.PROJECT_NAME || 'unrip') }}
|
||||||
PROJECT_REGISTRY_SECRET_NAME: ${{ vars.PROJECT_REGISTRY_SECRET_NAME || format('{0}-registry-creds', vars.PROJECT_NAME || 'unrip') }}
|
PROJECT_REGISTRY_SECRET_NAME: ${{ vars.PROJECT_REGISTRY_SECRET_NAME || format('{0}-registry-creds', vars.PROJECT_NAME || 'unrip') }}
|
||||||
REPO_CLONE_URL: ${{ github.server_url }}/${{ github.repository }}.git
|
REPO_CLONE_URL: ${{ github.server_url }}/${{ github.repository }}.git
|
||||||
steps:
|
steps:
|
||||||
|
|
@ -32,9 +33,13 @@ jobs:
|
||||||
run: |
|
run: |
|
||||||
IMAGE="$REGISTRY_HOST/$PROJECT_NAME:$IMAGE_TAG"
|
IMAGE="$REGISTRY_HOST/$PROJECT_NAME:$IMAGE_TAG"
|
||||||
BUILD_JOB="image-build-${GITHUB_SHA:0:12}"
|
BUILD_JOB="image-build-${GITHUB_SHA:0:12}"
|
||||||
|
BUILD_CONTEXT="$PROJECT_PATH"
|
||||||
|
DOCKERFILE_PATH="$PROJECT_PATH/Dockerfile"
|
||||||
{
|
{
|
||||||
echo "IMAGE=$IMAGE"
|
echo "IMAGE=$IMAGE"
|
||||||
echo "BUILD_JOB=$BUILD_JOB"
|
echo "BUILD_JOB=$BUILD_JOB"
|
||||||
|
echo "BUILD_CONTEXT=$BUILD_CONTEXT"
|
||||||
|
echo "DOCKERFILE_PATH=$DOCKERFILE_PATH"
|
||||||
echo "PROJECT_NAMESPACE=$PROJECT_NAMESPACE"
|
echo "PROJECT_NAMESPACE=$PROJECT_NAMESPACE"
|
||||||
echo "PROJECT_DEPLOYMENTS=$PROJECT_DEPLOYMENTS"
|
echo "PROJECT_DEPLOYMENTS=$PROJECT_DEPLOYMENTS"
|
||||||
echo "PROJECT_REGISTRY_SECRET_NAME=$PROJECT_REGISTRY_SECRET_NAME"
|
echo "PROJECT_REGISTRY_SECRET_NAME=$PROJECT_REGISTRY_SECRET_NAME"
|
||||||
|
|
@ -89,8 +94,8 @@ jobs:
|
||||||
- name: kaniko
|
- name: kaniko
|
||||||
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
image: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||||
args:
|
args:
|
||||||
- --context=/workspace
|
- --context=/workspace/${BUILD_CONTEXT}
|
||||||
- --dockerfile=/workspace/Dockerfile
|
- --dockerfile=/workspace/${DOCKERFILE_PATH}
|
||||||
- --destination=${IMAGE}
|
- --destination=${IMAGE}
|
||||||
- --cache=true
|
- --cache=true
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
|
|
|
||||||
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -4,6 +4,9 @@
|
||||||
__pycache__/
|
__pycache__/
|
||||||
*.pyc
|
*.pyc
|
||||||
.env
|
.env
|
||||||
|
**/.env
|
||||||
|
node_modules/
|
||||||
|
projects/**/node_modules/
|
||||||
deploy/k8s/overlays/hetzner-single-node/secrets/*.env
|
deploy/k8s/overlays/hetzner-single-node/secrets/*.env
|
||||||
deploy/k8s/overlays/hetzner-single-node/secrets/*.htpasswd
|
deploy/k8s/overlays/hetzner-single-node/secrets/*.htpasswd
|
||||||
!deploy/k8s/overlays/hetzner-single-node/secrets/*.example
|
!deploy/k8s/overlays/hetzner-single-node/secrets/*.example
|
||||||
|
|
|
||||||
240
README.md
240
README.md
|
|
@ -1,92 +1,75 @@
|
||||||
# near-intents-monitor
|
# near-intents-monitor platform repo
|
||||||
|
|
||||||
Production-shaped first slice of the trading system:
|
This repository now serves two roles:
|
||||||
|
|
||||||
- **venue ingest**: NEAR Intents solver-bus quote flow
|
1. **shared platform/infrastructure** for the Hetzner + k3s cluster
|
||||||
- **bus**: Redpanda first, Kafka-compatible by design
|
2. the embedded **`unrip` project**, isolated under `projects/unrip/` so it can later become its own repository
|
||||||
- **reactor**: dummy decision engine emitting commands
|
|
||||||
- **executor**: dummy execution worker with durable idempotency state
|
|
||||||
- **result consumer**: downstream observer of execution outcomes
|
|
||||||
|
|
||||||
## Canonical repo shape
|
## Repo layout
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
infra/
|
||||||
|
terraform/
|
||||||
|
hetzner/
|
||||||
|
scripts/
|
||||||
|
hetzner/
|
||||||
|
deploy/
|
||||||
|
hetzner/
|
||||||
|
k8s/
|
||||||
|
platform/
|
||||||
|
overlays/
|
||||||
|
hetzner-single-node/
|
||||||
|
projects/
|
||||||
|
unrip/
|
||||||
src/
|
src/
|
||||||
apps/
|
package.json
|
||||||
near-intents-ingest.mjs
|
|
||||||
dummy-reactor.mjs
|
|
||||||
dummy-executor.mjs
|
|
||||||
dummy-consumer.mjs
|
|
||||||
bus/
|
|
||||||
kafka/
|
|
||||||
producer.mjs
|
|
||||||
consumer.mjs
|
|
||||||
core/
|
|
||||||
event-envelope.mjs
|
|
||||||
executor-state-store.mjs
|
|
||||||
log.mjs
|
|
||||||
pair-filter.mjs
|
|
||||||
schemas.mjs
|
|
||||||
lib/
|
|
||||||
config.mjs
|
|
||||||
env.mjs
|
|
||||||
venues/
|
|
||||||
near-intents/
|
|
||||||
ingest.mjs
|
|
||||||
normalize.mjs
|
|
||||||
ws.mjs
|
|
||||||
compose.yml
|
|
||||||
Dockerfile
|
Dockerfile
|
||||||
docs/contracts.md
|
compose.yml
|
||||||
|
deploy/
|
||||||
|
k8s/
|
||||||
|
base/
|
||||||
|
docs/
|
||||||
```
|
```
|
||||||
|
|
||||||
## Event flow
|
## Shared platform at repo root
|
||||||
|
|
||||||
```text
|
Shared/root-owned parts include:
|
||||||
NEAR Intents WebSocket
|
- Hetzner Terraform
|
||||||
|
|
- cloud-init + bootstrap scripts
|
||||||
+--> raw.near_intents.quote
|
- cluster/platform Kubernetes manifests
|
||||||
|
|
|
||||||
v
|
|
||||||
norm.swap_demand
|
|
||||||
|
|
|
||||||
v
|
|
||||||
cmd.execute_trade
|
|
||||||
|
|
|
||||||
v
|
|
||||||
exec.trade_result
|
|
||||||
```
|
|
||||||
|
|
||||||
Core rule: services do not call each other directly for trading flow; they communicate through bus topics only.
|
|
||||||
|
|
||||||
## Contracts
|
|
||||||
See `docs/contracts.md`.
|
|
||||||
|
|
||||||
Current topics:
|
|
||||||
- `raw.near_intents.quote`
|
|
||||||
- `norm.swap_demand`
|
|
||||||
- `cmd.execute_trade`
|
|
||||||
- `exec.trade_result`
|
|
||||||
|
|
||||||
## Canonical deployment path
|
|
||||||
|
|
||||||
The canonical production path is the repo-driven Hetzner + k3s bootstrap flow.
|
|
||||||
Compose still exists for local development and optional single-machine testing, but it is not the primary production story.
|
|
||||||
|
|
||||||
Current single-node cluster stack includes:
|
|
||||||
- `unrip` workloads in namespace `unrip`
|
|
||||||
- Redpanda
|
|
||||||
- Forgejo
|
- Forgejo
|
||||||
- Forgejo runner
|
- Forgejo runner
|
||||||
- private registry
|
- registry
|
||||||
- cert-manager
|
- cert-manager
|
||||||
- Traefik via the k3s bundled ingress controller
|
- Traefik integration
|
||||||
- Grafana
|
- Grafana
|
||||||
- Loki
|
- Loki
|
||||||
- Promtail
|
- Promtail
|
||||||
- Headlamp
|
- Headlamp
|
||||||
|
- shared operator docs and runbooks
|
||||||
|
|
||||||
### Bootstrap entrypoint
|
## Embedded project: `unrip`
|
||||||
|
|
||||||
|
The trading-system code and project-specific deployment assets now live in:
|
||||||
|
|
||||||
|
- `projects/unrip/`
|
||||||
|
|
||||||
|
That directory contains:
|
||||||
|
- app source
|
||||||
|
- Node package files
|
||||||
|
- Docker build files
|
||||||
|
- local Compose setup
|
||||||
|
- project-specific Kubernetes manifests
|
||||||
|
- project-specific docs
|
||||||
|
|
||||||
|
Start there for project work:
|
||||||
|
- `projects/unrip/README.md`
|
||||||
|
- `projects/unrip/docs/contracts.md`
|
||||||
|
- `projects/unrip/docs/spec.md`
|
||||||
|
|
||||||
|
## Canonical production path
|
||||||
|
|
||||||
|
The canonical production path is the repo-driven Hetzner + k3s bootstrap flow.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cp scripts/hetzner/bootstrap-secrets.env.example scripts/hetzner/bootstrap-secrets.env
|
cp scripts/hetzner/bootstrap-secrets.env.example scripts/hetzner/bootstrap-secrets.env
|
||||||
|
|
@ -94,109 +77,50 @@ source scripts/hetzner/bootstrap-secrets.env
|
||||||
bash scripts/hetzner/bootstrap.sh
|
bash scripts/hetzner/bootstrap.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
The bootstrap script now:
|
Bootstrap now:
|
||||||
1. provisions or updates Hetzner infra with Terraform
|
1. provisions/updates Hetzner infra with Terraform
|
||||||
2. optionally manages DNS via Cloudflare or Porkbun
|
2. optionally manages DNS through Cloudflare or Porkbun
|
||||||
3. prefers Tailscale for admin/control-plane access when configured
|
3. fetches kubeconfig from the node into `.state/hetzner/kubeconfig.yaml`
|
||||||
4. fetches kubeconfig from the node into `.state/hetzner/kubeconfig.yaml`
|
4. renders `.state/hetzner/generated-overlay/`
|
||||||
5. renders `.state/hetzner/generated-overlay/` from repo manifests plus local secrets
|
5. applies shared platform manifests plus the selected project manifests
|
||||||
6. applies platform and project resources to k3s
|
6. bootstraps Forgejo admin, runner, repo, and Actions config
|
||||||
7. bootstraps Forgejo admin, runner, repo, and Actions configuration
|
7. seeds this repo into Forgejo
|
||||||
8. seeds this repo into Forgejo
|
8. lets Forgejo Actions perform the default image build + deploy path
|
||||||
9. lets Forgejo Actions perform the default build/push/deploy path
|
|
||||||
10. stores the generated Headlamp login token in `pass` when `HEADLAMP_ADMIN_TOKEN_PASS` is configured
|
|
||||||
|
|
||||||
Detailed bootstrap and destroy documentation lives in:
|
## Runtime surfaces
|
||||||
- `docs/hetzner-k3s-bootstrap.md`
|
|
||||||
- `docs/hetzner-self-hosted-ci-runbook.md`
|
|
||||||
- `docs/k8s-observability.md`
|
|
||||||
- `deploy/hetzner/README.md`
|
|
||||||
- `deploy/k8s/README.md`
|
|
||||||
- `deploy/k8s/overlays/hetzner-single-node/README.md`
|
|
||||||
|
|
||||||
### Runtime surfaces
|
|
||||||
|
|
||||||
- Forgejo: `https://git.doran.133011.xyz/`
|
- Forgejo: `https://git.doran.133011.xyz/`
|
||||||
- Registry: `https://registry.doran.133011.xyz/`
|
- Registry: `https://registry.doran.133011.xyz/`
|
||||||
- Grafana: `https://grafana.doran.133011.xyz/`
|
- Grafana: `https://grafana.doran.133011.xyz/`
|
||||||
- Headlamp: `https://headlamp.doran.133011.xyz/`
|
- Headlamp: `https://headlamp.doran.133011.xyz/`
|
||||||
|
|
||||||
### Operator notes
|
## Local project development
|
||||||
|
|
||||||
- Ingress is Traefik-based. The old ingress-nginx path is obsolete.
|
For the trading system itself:
|
||||||
- Grafana is for historical log search.
|
|
||||||
- Headlamp is for browsing workloads, pods, events, and pod logs.
|
|
||||||
- Use `pass`-backed `*_PASS` variables for secrets whenever possible.
|
|
||||||
|
|
||||||
## Executor persistence in k3s
|
|
||||||
|
|
||||||
The executor is stateful by design because it persists idempotency/execution tracking.
|
|
||||||
|
|
||||||
Current persistence boundary:
|
|
||||||
- app env uses `EXECUTOR_STATE_DIR=/var/lib/unrip/executor-state`
|
|
||||||
- in Kubernetes, the executor deployment mounts storage at that path
|
|
||||||
- the Hetzner single-node overlay pins storage to the k3s `local-path` storage class
|
|
||||||
|
|
||||||
Operational meaning:
|
|
||||||
- executor state lives on node-backed storage in the single-node k3s environment
|
|
||||||
- if that PVC or underlying node storage is lost, duplicate-suppression history is lost too
|
|
||||||
- treat executor persistence as part of the minimal durable state of the cluster
|
|
||||||
|
|
||||||
## Local development with Compose
|
|
||||||
|
|
||||||
Compose remains available for local development and debugging.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
cd projects/unrip
|
||||||
npm install
|
npm install
|
||||||
cp .env.example .env
|
cp .env.example .env
|
||||||
# edit .env
|
# edit .env
|
||||||
|
|
||||||
docker compose build
|
docker compose up -d --build
|
||||||
docker compose up -d
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Useful commands:
|
## Operator docs
|
||||||
|
|
||||||
```bash
|
Current operator/platform docs:
|
||||||
docker compose ps
|
- `docs/hetzner-k3s-bootstrap.md`
|
||||||
docker compose logs -f
|
- `docs/hetzner-self-hosted-ci-runbook.md`
|
||||||
docker compose logs -f near-intents-ingest dummy-reactor dummy-executor dummy-consumer
|
- `docs/k8s-observability.md`
|
||||||
docker compose restart dummy-executor
|
- `docs/hetzner-rebuild-pipeline.md`
|
||||||
docker compose down
|
- `deploy/hetzner/README.md`
|
||||||
docker compose down -v
|
- `deploy/k8s/README.md`
|
||||||
```
|
- `deploy/k8s/overlays/hetzner-single-node/README.md`
|
||||||
|
|
||||||
### Individual services
|
## Notes
|
||||||
```bash
|
|
||||||
npm run near-intents:ingest
|
|
||||||
npm run dummy-reactor
|
|
||||||
npm run dummy-executor
|
|
||||||
npm run dummy-consumer
|
|
||||||
```
|
|
||||||
|
|
||||||
Optional pair filter:
|
- Ingress is Traefik-based. The old ingress-nginx path is obsolete.
|
||||||
```bash
|
- Grafana is for historical log search.
|
||||||
npm run near-intents:ingest -- --pair 'asset_a->asset_b'
|
- Headlamp is for cluster/pod browsing and pod logs.
|
||||||
```
|
- Use `pass`-backed `*_PASS` values for secrets whenever possible.
|
||||||
|
|
||||||
## Idempotent executor behavior
|
|
||||||
- every command has a `command_id`
|
|
||||||
- commands carry `idempotency_key` and `execution_key`
|
|
||||||
- executor persists state under `EXECUTOR_STATE_DIR`
|
|
||||||
- completed commands are skipped after restart or replay
|
|
||||||
|
|
||||||
## Env
|
|
||||||
|
|
||||||
```env
|
|
||||||
NEAR_INTENTS_API_KEY=your_solver_jwt
|
|
||||||
NEAR_INTENTS_WS_URL=wss://solver-relay-v2.chaindefuser.com/ws
|
|
||||||
KAFKA_BROKERS=redpanda:9092
|
|
||||||
KAFKA_CLIENT_ID=unrip
|
|
||||||
KAFKA_TOPIC_RAW_NEAR_INTENTS_QUOTE=raw.near_intents.quote
|
|
||||||
KAFKA_TOPIC_NORM_SWAP_DEMAND=norm.swap_demand
|
|
||||||
KAFKA_TOPIC_CMD_EXECUTE_TRADE=cmd.execute_trade
|
|
||||||
KAFKA_TOPIC_EXEC_TRADE_RESULT=exec.trade_result
|
|
||||||
KAFKA_CONSUMER_GROUP_DUMMY=dummy-reactor-v1
|
|
||||||
KAFKA_CONSUMER_GROUP_EXECUTOR=dummy-executor-v1
|
|
||||||
EXECUTOR_STATE_DIR=/var/lib/unrip/executor-state
|
|
||||||
```
|
|
||||||
|
|
|
||||||
|
|
@ -3,11 +3,15 @@
|
||||||
This directory is the repo-driven deployment target for the single-node Hetzner+k3s bootstrap.
|
This directory is the repo-driven deployment target for the single-node Hetzner+k3s bootstrap.
|
||||||
|
|
||||||
## Layout
|
## Layout
|
||||||
- `base/` — shared bootstrap manifests plus the current `unrip` project manifests
|
- `base/` — compatibility kustomization that composes platform resources with the current `unrip` project path
|
||||||
- `projects/` — conventions for hosting multiple isolated projects on the same cluster
|
- `platform/` — shared cluster manifests
|
||||||
|
- `projects/` — naming/layout conventions for hosted projects
|
||||||
- `overlays/hetzner-single-node/` — first-node overlay with concrete hostnames, local-path storage, and generated secret references
|
- `overlays/hetzner-single-node/` — first-node overlay with concrete hostnames, local-path storage, and generated secret references
|
||||||
- `secrets/` — examples and instructions for supplying required secrets out-of-band
|
- `secrets/` — examples and instructions for supplying required secrets out-of-band
|
||||||
|
|
||||||
|
The actual `unrip` project manifests now live under:
|
||||||
|
- `projects/unrip/deploy/k8s/base/` at the repo root
|
||||||
|
|
||||||
## Shared cluster model
|
## Shared cluster model
|
||||||
Shared platform namespaces:
|
Shared platform namespaces:
|
||||||
- `forgejo`
|
- `forgejo`
|
||||||
|
|
|
||||||
|
|
@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
kind: Kustomization
|
kind: Kustomization
|
||||||
resources:
|
resources:
|
||||||
- ../platform/base
|
- ../platform/base
|
||||||
- ../projects/unrip/base
|
- ../../../projects/unrip/deploy/k8s/base
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
kind: Kustomization
|
kind: Kustomization
|
||||||
resources:
|
resources:
|
||||||
- ../../platform/base
|
- ../../platform/base
|
||||||
- ../../projects/unrip/base
|
- ../../../../projects/unrip/deploy/k8s/base
|
||||||
patches:
|
patches:
|
||||||
- path: ingress-hosts.patch.yaml
|
- path: ingress-hosts.patch.yaml
|
||||||
- path: issuer-email.patch.yaml
|
- path: issuer-email.patch.yaml
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,5 @@
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Namespace
|
kind: Namespace
|
||||||
metadata:
|
|
||||||
name: unrip
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/part-of: unrip
|
|
||||||
project.pi.io/type: project
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Namespace
|
|
||||||
metadata:
|
metadata:
|
||||||
name: forgejo
|
name: forgejo
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
# Projects on the shared cluster
|
# Projects on the shared cluster
|
||||||
|
|
||||||
This cluster is intended to host multiple independent projects.
|
This cluster is intended to host multiple independent projects.
|
||||||
|
This directory now documents project conventions; actual project code/manifests live under `/projects/<name>/` at the repo root.
|
||||||
|
|
||||||
## Pattern
|
## Pattern
|
||||||
- shared platform namespaces:
|
- shared platform namespaces:
|
||||||
|
|
@ -16,7 +17,7 @@ This cluster is intended to host multiple independent projects.
|
||||||
- future examples: `project-foo`, `project-bar`
|
- future examples: `project-foo`, `project-bar`
|
||||||
|
|
||||||
## How to add another project
|
## How to add another project
|
||||||
For each new project, create a project manifest set similar to `deploy/k8s/projects/unrip/base/`:
|
For each new project, create a project manifest set similar to `projects/unrip/deploy/k8s/base/` at the repo root:
|
||||||
- one namespace
|
- one namespace
|
||||||
- one project config map
|
- one project config map
|
||||||
- one secret name unique to the project
|
- one secret name unique to the project
|
||||||
|
|
@ -35,4 +36,4 @@ Recommended naming convention:
|
||||||
## Current project in this repo
|
## Current project in this repo
|
||||||
- project name: `unrip`
|
- project name: `unrip`
|
||||||
- namespace: `unrip`
|
- namespace: `unrip`
|
||||||
- project manifest: `deploy/k8s/projects/unrip/base/`
|
- project manifest: `projects/unrip/deploy/k8s/base/`
|
||||||
|
|
|
||||||
|
|
@ -1,85 +1,5 @@
|
||||||
# Event contracts
|
# Moved
|
||||||
|
|
||||||
## Envelope
|
This project-specific document moved to:
|
||||||
All bus messages use this envelope:
|
|
||||||
|
|
||||||
```json
|
- `projects/unrip/docs/contracts.md`
|
||||||
{
|
|
||||||
"event_id": "string",
|
|
||||||
"event_type": "string",
|
|
||||||
"venue": "string",
|
|
||||||
"source": "string|null",
|
|
||||||
"schema_version": 1,
|
|
||||||
"observed_at": "ISO-8601|null",
|
|
||||||
"ingested_at": "ISO-8601",
|
|
||||||
"payload": {},
|
|
||||||
"raw": {}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Topics
|
|
||||||
Current canonical topic set:
|
|
||||||
- `raw.near_intents.quote`
|
|
||||||
- `norm.swap_demand`
|
|
||||||
- `cmd.execute_trade`
|
|
||||||
- `exec.trade_result`
|
|
||||||
|
|
||||||
In Kubernetes bootstrap, Redpanda topic creation is currently handled by the repo-managed bootstrap job applied with the manifest set.
|
|
||||||
|
|
||||||
## `raw.near_intents.quote`
|
|
||||||
- `event_type`: `near_intents_quote_raw`
|
|
||||||
- `payload.message`: original venue-native payload
|
|
||||||
- `raw`: original venue-native payload
|
|
||||||
|
|
||||||
## `norm.swap_demand`
|
|
||||||
- `event_type`: `swap_demand`
|
|
||||||
- payload:
|
|
||||||
- `quote_id`
|
|
||||||
- `asset_in`
|
|
||||||
- `asset_out`
|
|
||||||
- `amount_in`
|
|
||||||
- `amount_out`
|
|
||||||
- `ttl_ms`
|
|
||||||
|
|
||||||
## `cmd.execute_trade`
|
|
||||||
- `event_type`: `execute_trade`
|
|
||||||
- payload:
|
|
||||||
- `command_id`
|
|
||||||
- `idempotency_key`
|
|
||||||
- `execution_key`
|
|
||||||
- `quote_id`
|
|
||||||
- `asset_in`
|
|
||||||
- `asset_out`
|
|
||||||
- `amount_in`
|
|
||||||
- `amount_out`
|
|
||||||
- `reason`
|
|
||||||
|
|
||||||
## `exec.trade_result`
|
|
||||||
- `event_type`: `trade_result`
|
|
||||||
- payload:
|
|
||||||
- `command_id`
|
|
||||||
- `idempotency_key`
|
|
||||||
- `execution_key`
|
|
||||||
- `quote_id`
|
|
||||||
- `status`
|
|
||||||
- `result_code`
|
|
||||||
- `note`
|
|
||||||
|
|
||||||
## Executor idempotency model
|
|
||||||
- `command_id` is unique per trade command and currently deterministic as `cmd-${quote_id}`
|
|
||||||
- `idempotency_key` is stable for semantic duplicate detection and currently `${venue}:${quote_id}`
|
|
||||||
- `execution_key` is the stable partition key and currently `${venue}:${asset_in}->${asset_out}`
|
|
||||||
- executor persists command state on durable storage before publishing a result
|
|
||||||
- already-completed `command_id`s are skipped on replay or restart
|
|
||||||
- if a command is seen again after a persisted `processing` state, the executor emits a recovered result path instead of blindly duplicating work
|
|
||||||
|
|
||||||
## Deployment and persistence implications
|
|
||||||
These contracts are tied to deployment behavior:
|
|
||||||
- executor duplicate suppression depends on durable persistence at `EXECUTOR_STATE_DIR`
|
|
||||||
- local Compose mounts that path for development/runtime testing
|
|
||||||
- the Hetzner single-node k3s path mounts persistent storage for the executor at `/var/lib/unrip/executor-state`
|
|
||||||
- in the current single-node target, that persistence is node-backed and should be treated as required operational state
|
|
||||||
|
|
||||||
Operational consequence:
|
|
||||||
- deleting the executor PVC or losing the node without migration discards idempotency history
|
|
||||||
- that can allow already-seen commands to be treated as new after recovery
|
|
||||||
|
|
|
||||||
|
|
@ -27,8 +27,10 @@ Goal: provision and deploy everything from this repo to a single Hetzner machine
|
||||||
|
|
||||||
## Files
|
## Files
|
||||||
- `infra/terraform/hetzner/`
|
- `infra/terraform/hetzner/`
|
||||||
- `deploy/k8s/base/`
|
- `deploy/k8s/platform/`
|
||||||
- `deploy/k8s/overlays/hetzner-single-node/`
|
- `deploy/k8s/overlays/hetzner-single-node/`
|
||||||
|
- `projects/unrip/deploy/k8s/base/`
|
||||||
|
- `projects/unrip/`
|
||||||
- `scripts/hetzner/bootstrap.sh`
|
- `scripts/hetzner/bootstrap.sh`
|
||||||
- `scripts/hetzner/configure-cloudflare-dns.sh`
|
- `scripts/hetzner/configure-cloudflare-dns.sh`
|
||||||
- `scripts/hetzner/destroy.sh`
|
- `scripts/hetzner/destroy.sh`
|
||||||
|
|
@ -104,6 +106,7 @@ Required values:
|
||||||
- `GRAFANA_ADMIN_PASSWORD_PASS` or `GRAFANA_ADMIN_PASSWORD`
|
- `GRAFANA_ADMIN_PASSWORD_PASS` or `GRAFANA_ADMIN_PASSWORD`
|
||||||
- optional `HEADLAMP_ADMIN_TOKEN_PASS` for storing the generated Headlamp login token back into `pass`
|
- optional `HEADLAMP_ADMIN_TOKEN_PASS` for storing the generated Headlamp login token back into `pass`
|
||||||
- optional repo settings: `FORGEJO_REPO_OWNER`, `FORGEJO_REPO_NAME`, `FORGEJO_REPO_PRIVATE`
|
- optional repo settings: `FORGEJO_REPO_OWNER`, `FORGEJO_REPO_NAME`, `FORGEJO_REPO_PRIVATE`
|
||||||
|
- optional project path settings: `PROJECT_DIR`, `PROJECT_KUSTOMIZE_PATH`
|
||||||
|
|
||||||
Optional for automatic DNS:
|
Optional for automatic DNS:
|
||||||
- Cloudflare:
|
- Cloudflare:
|
||||||
|
|
@ -187,7 +190,7 @@ Default bootstrap now automates the Forgejo handoff:
|
||||||
The workflow then:
|
The workflow then:
|
||||||
- starts a Kubernetes Job in the target namespace
|
- starts a Kubernetes Job in the target namespace
|
||||||
- checks out the repo inside that Job using the Forgejo job token via `Authorization: Bearer ...` HTTP auth
|
- checks out the repo inside that Job using the Forgejo job token via `Authorization: Bearer ...` HTTP auth
|
||||||
- uses Kaniko plus the Kubernetes registry auth secret to build and push `${REGISTRY_DOMAIN}/${PROJECT_NAME}:${GIT_SHA}`
|
- uses Kaniko plus the Kubernetes registry auth secret to build and push `${REGISTRY_DOMAIN}/${PROJECT_NAME}:${GIT_SHA}` from `PROJECT_PATH` inside the repo checkout
|
||||||
- updates the app deployments in `PROJECT_NAMESPACE`
|
- updates the app deployments in `PROJECT_NAMESPACE`
|
||||||
- waits for rollout
|
- waits for rollout
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -66,6 +66,7 @@ Bootstrap upserts these repository variables automatically:
|
||||||
- `PROJECT_NAME=${PROJECT_NAME}`
|
- `PROJECT_NAME=${PROJECT_NAME}`
|
||||||
- `PROJECT_NAMESPACE=${PROJECT_NAMESPACE}`
|
- `PROJECT_NAMESPACE=${PROJECT_NAMESPACE}`
|
||||||
- `PROJECT_DEPLOYMENTS` as a comma-separated version of the bootstrap deployment list
|
- `PROJECT_DEPLOYMENTS` as a comma-separated version of the bootstrap deployment list
|
||||||
|
- `PROJECT_PATH` as the repo-relative app directory used for Docker/Kaniko builds
|
||||||
|
|
||||||
The Forgejo repo configuration step is idempotent, so rerunning bootstrap updates the same repo secrets/variables in place.
|
The Forgejo repo configuration step is idempotent, so rerunning bootstrap updates the same repo secrets/variables in place.
|
||||||
|
|
||||||
|
|
@ -76,7 +77,7 @@ The workflow in `.forgejo/workflows/deploy.yml` now:
|
||||||
3. computes `IMAGE=${REGISTRY_HOST}/${PROJECT_NAME}:${GIT_SHA}`
|
3. computes `IMAGE=${REGISTRY_HOST}/${PROJECT_NAME}:${GIT_SHA}`
|
||||||
4. creates an in-cluster Kubernetes Job in `PROJECT_NAMESPACE`
|
4. creates an in-cluster Kubernetes Job in `PROJECT_NAMESPACE`
|
||||||
5. that Job checks out the repo with the Forgejo job token in an init container using an `Authorization: Bearer ...` header instead of embedding the token in the clone URL
|
5. that Job checks out the repo with the Forgejo job token in an init container using an `Authorization: Bearer ...` header instead of embedding the token in the clone URL
|
||||||
6. Kaniko builds and pushes the image using the Kubernetes registry auth secret
|
6. Kaniko builds and pushes the image from `PROJECT_PATH` using the Kubernetes registry auth secret
|
||||||
7. the workflow updates each deployment listed in `PROJECT_DEPLOYMENTS` inside `PROJECT_NAMESPACE`
|
7. the workflow updates each deployment listed in `PROJECT_DEPLOYMENTS` inside `PROJECT_NAMESPACE`
|
||||||
8. the workflow waits for rollout after each image update
|
8. the workflow waits for rollout after each image update
|
||||||
|
|
||||||
|
|
@ -84,6 +85,7 @@ Default behavior if you do not set project variables:
|
||||||
- `PROJECT_NAME=unrip`
|
- `PROJECT_NAME=unrip`
|
||||||
- `PROJECT_NAMESPACE=unrip`
|
- `PROJECT_NAMESPACE=unrip`
|
||||||
- `PROJECT_DEPLOYMENTS=near-intents-ingest,dummy-reactor,dummy-executor,dummy-consumer`
|
- `PROJECT_DEPLOYMENTS=near-intents-ingest,dummy-reactor,dummy-executor,dummy-consumer`
|
||||||
|
- `PROJECT_PATH=projects/unrip`
|
||||||
- `PROJECT_REGISTRY_SECRET_NAME=unrip-registry-creds`
|
- `PROJECT_REGISTRY_SECRET_NAME=unrip-registry-creds`
|
||||||
|
|
||||||
For a future project, reuse the same workflow by changing only the Forgejo repository variables instead of copying the workflow.
|
For a future project, reuse the same workflow by changing only the Forgejo repository variables instead of copying the workflow.
|
||||||
|
|
|
||||||
|
|
@ -1,198 +1,5 @@
|
||||||
# Minimal product: NEAR Intents demand monitor
|
# Moved
|
||||||
|
|
||||||
## Goal
|
This project-specific document moved to:
|
||||||
Build the smallest useful event-driven product for crypto trading research:
|
|
||||||
|
|
||||||
- read **live user demand** from NEAR Intents
|
- `projects/unrip/docs/minimal-product.md`
|
||||||
- publish demand into a **central Kafka/Redpanda-compatible bus**
|
|
||||||
- prove downstream consumption with a **dummy reactor**
|
|
||||||
- avoid dashboards, execution, wallets, storage, auth workflows beyond the required API key, strategy code, and generic infra beyond the message bus itself
|
|
||||||
|
|
||||||
## Why this is the right first slice
|
|
||||||
From the NEAR Intents docs, there are several possible data surfaces:
|
|
||||||
|
|
||||||
1. **Message Bus WebSocket `quote` subscription**
|
|
||||||
- Endpoint: `wss://solver-relay-v2.chaindefuser.com/ws`
|
|
||||||
- Real-time stream for quote requests
|
|
||||||
- Subscription request shape:
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"jsonrpc": "2.0",
|
|
||||||
"id": 1,
|
|
||||||
"method": "subscribe",
|
|
||||||
"params": ["quote"]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
- Expected live frame shape is JSON-RPC-like but should be treated as flexible. The adapter should accept quote payloads when the useful fields appear either:
|
|
||||||
- directly under `params`
|
|
||||||
- directly under `result`
|
|
||||||
- or at the top level of the message body
|
|
||||||
- Fields of interest include:
|
|
||||||
- `quote_id` (or equivalent request identifier)
|
|
||||||
- `defuse_asset_identifier_in`
|
|
||||||
- `defuse_asset_identifier_out`
|
|
||||||
- `exact_amount_in` or `exact_amount_out`
|
|
||||||
- `min_deadline_ms`
|
|
||||||
- Subscription acknowledgements may also vary. They may arrive as an `id`-matched JSON-RPC response with a simple `result`, a structured `result`, or other non-quote control frame before the first quote event.
|
|
||||||
- This is the closest public signal to **current demand**.
|
|
||||||
|
|
||||||
2. **Message Bus JSON-RPC `publish_intent` / `get_status`**
|
|
||||||
- Endpoint: `https://solver-relay-v2.chaindefuser.com/rpc`
|
|
||||||
- Useful for posting intents or checking a known `intent_hash`
|
|
||||||
- Not a public firehose of all intents.
|
|
||||||
|
|
||||||
3. **Explorer API `/api/v0/transactions`**
|
|
||||||
- Historical and analytics friendly
|
|
||||||
- Requires JWT auth
|
|
||||||
- Better for history, not best for a minimal live monitor
|
|
||||||
|
|
||||||
4. **Verifier contract intent payloads**
|
|
||||||
- The on-chain swap expression is usually `token_diff`
|
|
||||||
- Important for understanding settlement semantics
|
|
||||||
- Not the easiest first live intake path for a lean bus-first system
|
|
||||||
|
|
||||||
## Product decision
|
|
||||||
The minimal product should monitor **WebSocket `quote` events** and route them through a bus-first runtime.
|
|
||||||
|
|
||||||
### Why
|
|
||||||
- closest live signal to user demand
|
|
||||||
- directly reflects what users are requesting from solvers
|
|
||||||
- enough to answer the first trading question: **what assets are being requested right now?**
|
|
||||||
- decouples venue intake from downstream analysis through Kafka-compatible topics
|
|
||||||
|
|
||||||
### Important implementation note
|
|
||||||
Current docs for the market-maker quickstart and live endpoint behavior indicate the Message Bus requires a **partner API key / JWT** in the `Authorization: Bearer ...` header.
|
|
||||||
That means the best path is still the quote stream, but live operation is partner-gated.
|
|
||||||
|
|
||||||
### Important caveat
|
|
||||||
A `quote` event is **pre-trade demand**, not guaranteed execution.
|
|
||||||
That is fine for v0. The purpose is demand sensing, not settlement accounting.
|
|
||||||
|
|
||||||
## Runtime shape
|
|
||||||
|
|
||||||
```text
|
|
||||||
NEAR Intents websocket
|
|
||||||
|
|
|
||||||
v
|
|
||||||
src/apps/near-intents-ingest.mjs
|
|
||||||
|
|
|
||||||
+--> raw.near_intents.quote
|
|
||||||
|
|
|
||||||
+--> norm.swap_demand
|
|
||||||
|
|
|
||||||
v
|
|
||||||
src/apps/dummy-consumer.mjs
|
|
||||||
```
|
|
||||||
|
|
||||||
### Runtime contracts
|
|
||||||
|
|
||||||
#### Ingest app
|
|
||||||
`src/apps/near-intents-ingest.mjs`:
|
|
||||||
- loads env
|
|
||||||
- parses optional `--pair 'asset_a->asset_b'`
|
|
||||||
- starts the NEAR Intents websocket adapter
|
|
||||||
- writes raw and normalized events to the configured broker
|
|
||||||
|
|
||||||
#### Dummy consumer
|
|
||||||
`src/apps/dummy-consumer.mjs`:
|
|
||||||
- subscribes to `norm.swap_demand`
|
|
||||||
- logs observed pair and quote id
|
|
||||||
- exists only to prove a downstream consumer contract
|
|
||||||
|
|
||||||
#### Bus config
|
|
||||||
Default env-driven topics and group ids:
|
|
||||||
- `KAFKA_TOPIC_RAW_NEAR_INTENTS_QUOTE=raw.near_intents.quote`
|
|
||||||
- `KAFKA_TOPIC_NORM_SWAP_DEMAND=norm.swap_demand`
|
|
||||||
- `KAFKA_CONSUMER_GROUP_DUMMY=dummy-reactor-v1`
|
|
||||||
|
|
||||||
Redpanda is a valid runtime target because the transport is Kafka-compatible.
|
|
||||||
|
|
||||||
## Internal model
|
|
||||||
Normalize each quote event into a thin bus envelope:
|
|
||||||
|
|
||||||
Top-level envelope fields:
|
|
||||||
- `venue`
|
|
||||||
- `source`
|
|
||||||
- `type`
|
|
||||||
- `eventId`
|
|
||||||
- `occurredAt`
|
|
||||||
- `ingestedAt`
|
|
||||||
- `assetIn`
|
|
||||||
- `assetOut`
|
|
||||||
- `raw`
|
|
||||||
- `quote`
|
|
||||||
|
|
||||||
Nested `quote` fields:
|
|
||||||
- `quoteId`
|
|
||||||
- `assetIn`
|
|
||||||
- `assetOut`
|
|
||||||
- `amountIn`
|
|
||||||
- `amountOut`
|
|
||||||
- `ttlMs`
|
|
||||||
|
|
||||||
Field extraction must remain tolerant to known upstream aliases, and normalization should continue to operate on the merged `metadata + data` payload shape from the Message Bus event.
|
|
||||||
The live adapter now intentionally accepts quote-like payloads from `params`, `result`, or the top-level message body, but only processes frames that actually look like quote data. Subscription acknowledgements and unrelated control frames should still be ignored.
|
|
||||||
|
|
||||||
## Filtering
|
|
||||||
The ingest runtime supports an optional exact-pair filter:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm run near-intents:ingest -- --pair 'asset_a->asset_b'
|
|
||||||
```
|
|
||||||
|
|
||||||
The filter is direction-agnostic, so the reversed asset order is also accepted.
|
|
||||||
|
|
||||||
## Scope boundaries
|
|
||||||
### Must do
|
|
||||||
- connect to the websocket
|
|
||||||
- subscribe to `quote` and tolerate control frames
|
|
||||||
- normalize quote events into one compact model
|
|
||||||
- publish raw and normalized events to Kafka/Redpanda-compatible topics
|
|
||||||
- allow a downstream consumer to react to normalized events
|
|
||||||
- reconnect automatically on disconnect
|
|
||||||
- document `npm` and `node` entrypoints
|
|
||||||
|
|
||||||
### Must not do
|
|
||||||
- Python packaging or CLI guidance
|
|
||||||
- TUI-specific product requirements
|
|
||||||
- charts
|
|
||||||
- account details
|
|
||||||
- pnl
|
|
||||||
- routing internals
|
|
||||||
- market making controls
|
|
||||||
- execution buttons
|
|
||||||
- config panels
|
|
||||||
- speculative infra beyond the current bus and dummy consumer
|
|
||||||
|
|
||||||
## Path to success
|
|
||||||
1. Connect to WebSocket
|
|
||||||
2. Subscribe to `quote`
|
|
||||||
3. Normalize incoming events into one compact model
|
|
||||||
4. Publish raw envelopes to `raw.near_intents.quote`
|
|
||||||
5. Publish normalized envelopes to `norm.swap_demand`
|
|
||||||
6. Start a dummy consumer on the normalized topic
|
|
||||||
7. Reconnect automatically on disconnect
|
|
||||||
8. Only after this works, consider:
|
|
||||||
- `quote_status`-specific downstream handling
|
|
||||||
- historical replay via Explorer API
|
|
||||||
- token metadata enrichment
|
|
||||||
- filtering and alerts beyond `--pair`
|
|
||||||
|
|
||||||
## Packaging alignment
|
|
||||||
Current repository packaging and usage should stay aligned around the JavaScript runtime entrypoints:
|
|
||||||
|
|
||||||
- package scripts:
|
|
||||||
- `npm run near-intents:ingest`
|
|
||||||
- `npm run dummy-consumer`
|
|
||||||
- `npm start` as a compatibility wrapper
|
|
||||||
- direct app entrypoints:
|
|
||||||
- `node src/apps/near-intents-ingest.mjs`
|
|
||||||
- `node src/apps/dummy-consumer.mjs`
|
|
||||||
|
|
||||||
Documentation should treat the npm scripts and `src/apps/*` node entrypoints as canonical. Older single-file and Python/TUI instructions should remain removed to avoid runtime confusion.
|
|
||||||
|
|
||||||
## Sources
|
|
||||||
- NEAR Intents Message Bus WebSocket docs: `subscribe` with `quote` / `quote_status`
|
|
||||||
- NEAR Intents Message Bus RPC docs: `quote`, `publish_intent`, `get_status`
|
|
||||||
- Verifier contract docs: `token_diff` intent type
|
|
||||||
- Explorer API OpenAPI: authenticated historical transactions
|
|
||||||
|
|
|
||||||
|
|
@ -1,383 +1,5 @@
|
||||||
# Trading System Architecture Notes for Next Session
|
# Moved
|
||||||
|
|
||||||
## Objective
|
This project-specific document moved to:
|
||||||
Build the first real version of the trading system as an event-driven, multi-service architecture.
|
|
||||||
|
|
||||||
Current implemented seed:
|
- `projects/unrip/docs/next-session-architecture.md`
|
||||||
- NEAR Intents ingest in Node.js
|
|
||||||
- Kafka-compatible bus usage via `kafkajs`
|
|
||||||
- dummy reactor / executor / result consumer loop
|
|
||||||
|
|
||||||
Next session should continue from this architecture, not revert to a monolith, local-only script, or TUI.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Core Architecture
|
|
||||||
All components are independent services.
|
|
||||||
They communicate only through a central Kafka-compatible bus (Redpanda first, Kafka-compatible by design).
|
|
||||||
|
|
||||||
### Service classes
|
|
||||||
- venue ingestors
|
|
||||||
- normalizers
|
|
||||||
- reactors / decision engines
|
|
||||||
- executors
|
|
||||||
- downstream consumers / monitors / archivers / replay tools
|
|
||||||
|
|
||||||
### Service communication rule
|
|
||||||
No direct service-to-service calls for core trading flow.
|
|
||||||
Use bus topics only.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Venue-Oriented Structure
|
|
||||||
The system should be organized by venue.
|
|
||||||
Each venue can have different:
|
|
||||||
- ingest/feed mechanics
|
|
||||||
- normalization logic
|
|
||||||
- execution mechanics
|
|
||||||
|
|
||||||
### Per-venue responsibilities
|
|
||||||
- `ingest` = venue-native intake
|
|
||||||
- `normalize` = convert venue-native payload into canonical internal event
|
|
||||||
- `execute` = venue-specific action logic
|
|
||||||
|
|
||||||
Planned shape:
|
|
||||||
```text
|
|
||||||
src/
|
|
||||||
apps/
|
|
||||||
bus/
|
|
||||||
core/
|
|
||||||
venues/
|
|
||||||
near-intents/
|
|
||||||
ingest
|
|
||||||
normalize
|
|
||||||
execute
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Bus Choice
|
|
||||||
Use **Redpanda** first, but stay fully **Kafka-compatible**.
|
|
||||||
|
|
||||||
### Reason
|
|
||||||
Requirements:
|
|
||||||
- high throughput
|
|
||||||
- low latency
|
|
||||||
- retention
|
|
||||||
- replay
|
|
||||||
- multiple producers/consumers
|
|
||||||
- independent services
|
|
||||||
- future scale-out
|
|
||||||
- multi-language compatibility
|
|
||||||
|
|
||||||
### Constraint
|
|
||||||
Do not use broker-specific features that make migration to Kafka difficult.
|
|
||||||
Use standard Kafka clients and semantics.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Data Model Principles
|
|
||||||
Kafka/Redpanda is the operational event backbone.
|
|
||||||
|
|
||||||
### Event model rules
|
|
||||||
- append-only
|
|
||||||
- immutable events
|
|
||||||
- versioned schemas
|
|
||||||
- raw and normalized events both preserved
|
|
||||||
|
|
||||||
### Every event should include
|
|
||||||
- `event_id`
|
|
||||||
- `event_type`
|
|
||||||
- `venue`
|
|
||||||
- `observed_at` / `ingested_at`
|
|
||||||
- `schema_version`
|
|
||||||
- `payload`
|
|
||||||
- optionally raw/original payload where appropriate
|
|
||||||
|
|
||||||
### Raw vs normalized
|
|
||||||
Keep both.
|
|
||||||
- raw topics = exact venue-native source truth
|
|
||||||
- normalized topics = canonical research/trading inputs
|
|
||||||
|
|
||||||
This is required for:
|
|
||||||
- replay
|
|
||||||
- debugging
|
|
||||||
- future backtesting
|
|
||||||
- future Spark/batch processing
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Current/Planned Topic Flow
|
|
||||||
Minimal 3-stage pipeline:
|
|
||||||
|
|
||||||
1. ingest publishes normalized demand
|
|
||||||
2. reactor publishes trade command
|
|
||||||
3. executor publishes trade result
|
|
||||||
|
|
||||||
### Topic classes
|
|
||||||
- `raw.*` = raw venue-native events
|
|
||||||
- `norm.*` = canonical normalized market events
|
|
||||||
- `cmd.*` = execution commands
|
|
||||||
- `exec.*` = execution outcomes
|
|
||||||
- later `signal.*` if needed for reactor outputs before command stage
|
|
||||||
|
|
||||||
### Current minimal topics
|
|
||||||
- `norm.swap_demand`
|
|
||||||
- `cmd.execute_trade`
|
|
||||||
- `exec.trade_result`
|
|
||||||
|
|
||||||
### NEAR Intents
|
|
||||||
NEAR Intents source currently feeds quote-demand style events from solver-bus websocket.
|
|
||||||
This is a venue ingest source, not the whole trading system.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Execution Safety / Zero Downtime Requirements
|
|
||||||
This is critical.
|
|
||||||
|
|
||||||
### Constraint
|
|
||||||
Multiple executors must never duplicate the same trade/action during deploys, restarts, or rebalances.
|
|
||||||
|
|
||||||
### Must-have rules
|
|
||||||
1. Every execution command must carry a unique `command_id`
|
|
||||||
2. Commands must include deterministic idempotency information
|
|
||||||
3. Executors must be idempotent
|
|
||||||
4. Executors must belong to a consumer group per executor role
|
|
||||||
5. Commands should be partitioned by a stable execution key where ordering matters
|
|
||||||
6. Executor state must be persisted durably enough to detect duplicate command execution
|
|
||||||
|
|
||||||
### Kafka consumer groups are not sufficient alone
|
|
||||||
They help assign work, but they do not guarantee no duplicate processing under restart/rebalance conditions.
|
|
||||||
Idempotency is still required.
|
|
||||||
|
|
||||||
### Rolling updates / zero downtime
|
|
||||||
Executors must support:
|
|
||||||
- graceful shutdown
|
|
||||||
- stop taking new work before exit
|
|
||||||
- finish or safely recover in-flight work
|
|
||||||
- commit offsets only after safe execution state transition
|
|
||||||
|
|
||||||
### Persistence implication
|
|
||||||
Executor idempotency state is not optional metadata.
|
|
||||||
It is operational state that must survive pod restarts.
|
|
||||||
|
|
||||||
Current single-node k3s direction:
|
|
||||||
- executor state lives at `/var/lib/unrip/executor-state`
|
|
||||||
- Kubernetes mounts that path through persistent storage
|
|
||||||
- the Hetzner single-node overlay currently targets k3s `local-path` storage
|
|
||||||
- node loss without storage migration means duplicate-suppression history is lost
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Deployment Target
|
|
||||||
### First deployment phase
|
|
||||||
- single machine on Hetzner
|
|
||||||
- but still multiple independent services
|
|
||||||
- no architecture shortcuts that prevent future clustering
|
|
||||||
|
|
||||||
### Future target
|
|
||||||
- split across multiple machines
|
|
||||||
- cluster capable
|
|
||||||
- fault tolerant
|
|
||||||
- multi-node
|
|
||||||
- zero-downtime deploys
|
|
||||||
|
|
||||||
### Deployment rules from day 1
|
|
||||||
- every component is a separate container/service
|
|
||||||
- all config via env/config files
|
|
||||||
- communication over network/bus only
|
|
||||||
- persistent components use mounted volumes/PVCs
|
|
||||||
- no manual SSH-based operational workflow
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Infrastructure / Ops Direction
|
|
||||||
Target environment:
|
|
||||||
- Hetzner
|
|
||||||
- self-hosted CI/CD
|
|
||||||
- provisioning by code
|
|
||||||
- no GitHub dependency
|
|
||||||
|
|
||||||
### Desired stack direction
|
|
||||||
- Terraform for Hetzner provisioning
|
|
||||||
- Kubernetes-oriented target from the start
|
|
||||||
- self-hosted Git + CI/CD
|
|
||||||
- Kafka-compatible broker
|
|
||||||
- object storage later for long-term archived event history
|
|
||||||
|
|
||||||
### Single-node first, future cluster later
|
|
||||||
The first version may run on one machine, but deployment structure should already match a future distributed system.
|
|
||||||
|
|
||||||
### Current canonical operator path
|
|
||||||
The repo now documents and partially implements this path as the primary deployment workflow:
|
|
||||||
|
|
||||||
#### Phase 0: workstation bootstrap
|
|
||||||
1. A local operator workstation prepares bootstrap secrets in `scripts/hetzner/bootstrap-secrets.env`.
|
|
||||||
2. The operator runs `bash scripts/hetzner/bootstrap.sh`.
|
|
||||||
3. Terraform provisions the server, firewall, network, and cloud-init user-data.
|
|
||||||
4. cloud-init installs k3s automatically and prepares persistence directories plus bootstrap artifacts.
|
|
||||||
5. The workstation waits for the public k3s API endpoint to report ready.
|
|
||||||
6. The workstation writes `.state/hetzner/kubeconfig.yaml`.
|
|
||||||
7. The workstation injects initial Kubernetes Secrets for app and Forgejo bootstrap.
|
|
||||||
8. The workstation applies repo-managed Kubernetes manifests under `deploy/k8s/`.
|
|
||||||
9. The workstation performs the first image/bootstrap delivery attempt for the app workloads.
|
|
||||||
10. The workstation verifies rollout status.
|
|
||||||
|
|
||||||
#### Phase 1: self-hosted handoff
|
|
||||||
1. Forgejo becomes reachable in-cluster.
|
|
||||||
2. The operator completes initial Forgejo admin/repo setup.
|
|
||||||
3. This repo is pushed or mirrored into Forgejo.
|
|
||||||
4. The Forgejo runner becomes the routine app deployment mechanism.
|
|
||||||
5. Terraform remains the infra mutation entrypoint unless further automated later.
|
|
||||||
|
|
||||||
### Failure-recovery expectation
|
|
||||||
The bootstrap path must be rerunnable from the workstation.
|
|
||||||
Docs should keep treating recovery as:
|
|
||||||
- fix local secrets/inputs
|
|
||||||
- rerun the bootstrap script
|
|
||||||
- inspect the cluster with the generated kubeconfig
|
|
||||||
- destroy/recreate infra with `scripts/hetzner/destroy.sh` only when required
|
|
||||||
|
|
||||||
### Current repo-state caveats
|
|
||||||
The direction is clear, but the implementation is still mid-transition:
|
|
||||||
- the bootstrap script currently applies `deploy/k8s/base` directly rather than the Hetzner overlay
|
|
||||||
- kubeconfig/auth handling is not yet fully production-hardened
|
|
||||||
- first image delivery is still a bootstrap workaround rather than a final registry-native CI path
|
|
||||||
- Forgejo admin bootstrap, repo creation, and Actions configuration still require operator steps
|
|
||||||
- local Compose remains in the repo for development/testing, not as the canonical production path
|
|
||||||
|
|
||||||
### Minimal repo layout target
|
|
||||||
```text
|
|
||||||
deploy/
|
|
||||||
hetzner/
|
|
||||||
README.md
|
|
||||||
k8s/
|
|
||||||
base/
|
|
||||||
overlays/
|
|
||||||
hetzner-single-node/
|
|
||||||
infra/
|
|
||||||
terraform/
|
|
||||||
hetzner/
|
|
||||||
```
|
|
||||||
|
|
||||||
Guidelines:
|
|
||||||
- `infra/terraform/hetzner/` owns VM, firewall, networking, and cloud-init rendering
|
|
||||||
- `deploy/k8s/` owns Kubernetes-native manifests and overlays
|
|
||||||
- app runtime manifests should remain Kubernetes-native so they can later move from single-node k3s to a larger cluster with minimal rewrite
|
|
||||||
- secret material must not live in git in plaintext; bootstrap docs should describe workstation-driven injection or generated secret references
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Local Development / Testing Direction
|
|
||||||
Do not assume manual multi-terminal operation long term.
|
|
||||||
|
|
||||||
### Requirement
|
|
||||||
Need an orchestrated local/dev runtime.
|
|
||||||
|
|
||||||
### Local dev should preserve real boundaries
|
|
||||||
- separate services
|
|
||||||
- broker present
|
|
||||||
- env/config driven
|
|
||||||
- same event flow as production
|
|
||||||
|
|
||||||
### Current local/dev answer
|
|
||||||
Compose is still acceptable for:
|
|
||||||
- developer laptops
|
|
||||||
- fast local iteration
|
|
||||||
- debugging event flow
|
|
||||||
- validating container boundaries before Kubernetes rollout
|
|
||||||
|
|
||||||
But Compose should remain explicitly secondary to the repo-driven Hetzner + k3s path for production operations.
|
|
||||||
|
|
||||||
### Testing layers
|
|
||||||
1. unit tests for normalizers / schema logic / helpers
|
|
||||||
2. integration tests against Kafka-compatible broker
|
|
||||||
3. replay/simulation tests using retained event streams
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Spark Readiness
|
|
||||||
Do not add Spark now.
|
|
||||||
But keep the system Spark-compatible later by:
|
|
||||||
- preserving raw events
|
|
||||||
- preserving normalized events
|
|
||||||
- using immutable append-only event streams
|
|
||||||
- versioning schemas
|
|
||||||
- separating operational event log from future analytical processing
|
|
||||||
|
|
||||||
Spark later would be for:
|
|
||||||
- large-scale backtesting
|
|
||||||
- feature generation
|
|
||||||
- archive processing
|
|
||||||
- multi-venue analytics
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Immediate Next Engineering Tasks
|
|
||||||
Next session should focus on the following.
|
|
||||||
|
|
||||||
### 1. Clean current repo structure
|
|
||||||
Remove duplicate/legacy paths and keep one canonical structure only.
|
|
||||||
|
|
||||||
### 2. Keep/complete the 3-stage loop
|
|
||||||
- NEAR Intents ingest -> `norm.swap_demand`
|
|
||||||
- dummy reactor -> `cmd.execute_trade`
|
|
||||||
- dummy executor -> `exec.trade_result`
|
|
||||||
- downstream result consumer
|
|
||||||
|
|
||||||
### 3. Define canonical schemas
|
|
||||||
Define concrete event schemas for:
|
|
||||||
- normalized swap demand
|
|
||||||
- execute trade command
|
|
||||||
- trade result
|
|
||||||
|
|
||||||
### 4. Define executor idempotency model
|
|
||||||
Specify:
|
|
||||||
- `command_id`
|
|
||||||
- idempotency key rules
|
|
||||||
- execution state transition rules
|
|
||||||
- duplicate handling rules
|
|
||||||
|
|
||||||
### 5. Move toward production-shaped deployment
|
|
||||||
Design for:
|
|
||||||
- one service per container
|
|
||||||
- single-node deployment first
|
|
||||||
- future multi-node split without app rewrite
|
|
||||||
|
|
||||||
### 6. Harden provisioning/deployment path
|
|
||||||
Next infra work should continue improving:
|
|
||||||
- Hetzner provisioning by code
|
|
||||||
- workstation bootstrap rerunnability
|
|
||||||
- self-hosted CI/CD handoff
|
|
||||||
- registry-native image delivery
|
|
||||||
- overlay convergence for the Hetzner single-node target
|
|
||||||
|
|
||||||
Status update:
|
|
||||||
- minimal Terraform exists under `infra/terraform/hetzner`
|
|
||||||
- first boot is cloud-init driven and installs k3s automatically
|
|
||||||
- bootstrap now starts from a local operator workstation rather than manual host login
|
|
||||||
- Kubernetes assets exist under `deploy/k8s`
|
|
||||||
- executor persistence boundaries are explicit for single-node k3s
|
|
||||||
- self-hosted CI handoff is documented, but still requires follow-up hardening
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Non-Goals for Next Session
|
|
||||||
- no dashboards
|
|
||||||
- no UI/TUI
|
|
||||||
- no monolith convenience architecture
|
|
||||||
- no SQLite-first system of record
|
|
||||||
- no direct coupling between ingest, decision, and execution
|
|
||||||
- no temporary local-only shortcuts that block future cluster deployment
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Guiding Principle
|
|
||||||
Build the single-node first version as if it is already a distributed system:
|
|
||||||
- separate services
|
|
||||||
- durable event bus
|
|
||||||
- replayable events
|
|
||||||
- explicit contracts
|
|
||||||
- idempotent execution
|
|
||||||
- production-compatible deployment boundaries
|
|
||||||
- bootstrapable from scratch without manual SSH-based host setup
|
|
||||||
|
|
|
||||||
145
docs/spec.md
145
docs/spec.md
|
|
@ -1,144 +1,5 @@
|
||||||
# NEAR Intents demand monitor: bus-first source plan
|
# Moved
|
||||||
|
|
||||||
## Why websocket quote requests are still the MVP demand signal
|
This project-specific document moved to:
|
||||||
|
|
||||||
Public solver quote requests remain the closest thing to live demand because they appear when a user or integration asks the network for executable pricing. They are still the right upstream source, but the runtime architecture is now bus-first rather than terminal-first.
|
- `projects/unrip/docs/spec.md`
|
||||||
|
|
||||||
Why this source wins for a first monitor:
|
|
||||||
|
|
||||||
- **Most real-time:** quote requests arrive before settlement and usually before a completed trade is visible anywhere else.
|
|
||||||
- **Closer to intent formation:** they reflect active user demand, not just historical outcomes.
|
|
||||||
- **Operationally simple:** a single websocket feed can drive the ingest side without indexing chains, scraping dashboards, or correlating multiple APIs.
|
|
||||||
- **Good enough for ranking demand:** even if quotes do not always become fills, repeated quote flow is still a strong indicator of what users are currently trying to do.
|
|
||||||
|
|
||||||
## Tradeoffs vs other sources
|
|
||||||
|
|
||||||
### Solver websocket quote requests
|
|
||||||
|
|
||||||
Pros:
|
|
||||||
- lowest-latency view of current demand
|
|
||||||
- directly tied to solver workflow
|
|
||||||
- suitable for a streaming ingest adapter
|
|
||||||
- can be normalized into pair, size, and frequency metrics immediately
|
|
||||||
|
|
||||||
Cons:
|
|
||||||
- quote requests are **interest**, not guaranteed executed volume
|
|
||||||
- public access may still be rate-limited, undocumented, or require credentials depending on environment
|
|
||||||
- schema and availability may change faster than user-facing products
|
|
||||||
|
|
||||||
### Explorer
|
|
||||||
|
|
||||||
Explorer (`https://explorer.near-intents.org/`) is useful for validation and historical inspection, but it is usually a worse primary source for an MVP demand monitor.
|
|
||||||
|
|
||||||
Tradeoffs:
|
|
||||||
- better for human inspection than low-latency streaming
|
|
||||||
- likely shows processed/published activity instead of raw quote demand
|
|
||||||
- may lag the actual request path
|
|
||||||
- less convenient as a machine-first demand feed
|
|
||||||
|
|
||||||
### Status dashboard / published status
|
|
||||||
|
|
||||||
Status (`https://status.near-intents.org/posts/dashboard`) is useful for system health, not demand discovery.
|
|
||||||
|
|
||||||
Tradeoffs:
|
|
||||||
- tells us whether the platform is up, degraded, or incident-affected
|
|
||||||
- does **not** represent per-request user demand
|
|
||||||
- coarse and aggregated by design
|
|
||||||
|
|
||||||
### Published intents / settled outcomes
|
|
||||||
|
|
||||||
Published or completed intents are higher-confidence signals, but lower-fidelity for immediate demand sensing.
|
|
||||||
|
|
||||||
Tradeoffs:
|
|
||||||
- stronger evidence of actual execution
|
|
||||||
- misses abandoned demand and pre-trade discovery
|
|
||||||
- arrives later than quote traffic
|
|
||||||
- may require more indexing and entity correlation work
|
|
||||||
|
|
||||||
## Runtime architecture
|
|
||||||
|
|
||||||
```text
|
|
||||||
solver websocket quote stream
|
|
||||||
|
|
|
||||||
v
|
|
||||||
src/apps/near-intents-ingest.mjs
|
|
||||||
|
|
|
||||||
+--> raw.near_intents.quote
|
|
||||||
|
|
|
||||||
+--> norm.swap_demand
|
|
||||||
|
|
|
||||||
v
|
|
||||||
src/apps/dummy-consumer.mjs
|
|
||||||
```
|
|
||||||
|
|
||||||
### Responsibilities
|
|
||||||
|
|
||||||
#### `src/apps/near-intents-ingest.mjs`
|
|
||||||
- loads env from `.env`
|
|
||||||
- parses optional `--pair 'asset_a->asset_b'`
|
|
||||||
- connects to the NEAR Intents websocket
|
|
||||||
- subscribes to `quote` and `quote_status`
|
|
||||||
- publishes raw venue envelopes to `raw.near_intents.quote`
|
|
||||||
- publishes normalized swap-demand envelopes to `norm.swap_demand`
|
|
||||||
|
|
||||||
#### `src/apps/dummy-consumer.mjs`
|
|
||||||
- consumes normalized events from `norm.swap_demand`
|
|
||||||
- logs observed demand as a placeholder for later strategy logic
|
|
||||||
|
|
||||||
#### Kafka / Redpanda layer
|
|
||||||
- broker endpoint comes from `KAFKA_BROKERS`
|
|
||||||
- Redpanda is supported through Kafka protocol compatibility
|
|
||||||
- topics are configurable via env and default to:
|
|
||||||
- `raw.near_intents.quote`
|
|
||||||
- `norm.swap_demand`
|
|
||||||
|
|
||||||
## Assumptions and limitations
|
|
||||||
|
|
||||||
- The websocket is the best available **MVP** source, not a perfect truth source.
|
|
||||||
- Demand is approximated by quote requests, not by settled intents.
|
|
||||||
- Live endpoints require auth in practice; `NEAR_INTENTS_API_KEY` must be provided.
|
|
||||||
- Request schemas may evolve; the parser should tolerate missing fields.
|
|
||||||
- The current product is intentionally minimal: no database, no backfill, no reconciliation against chain state.
|
|
||||||
- The dummy consumer proves the decoupled flow but is not a strategy engine.
|
|
||||||
|
|
||||||
## Run instructions
|
|
||||||
|
|
||||||
Install:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm install
|
|
||||||
```
|
|
||||||
|
|
||||||
Start ingest:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm run near-intents:ingest
|
|
||||||
```
|
|
||||||
|
|
||||||
Direct node entrypoint:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
node src/apps/near-intents-ingest.mjs
|
|
||||||
```
|
|
||||||
|
|
||||||
Run with exact-pair filtering:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm run near-intents:ingest -- --pair 'asset_a->asset_b'
|
|
||||||
```
|
|
||||||
|
|
||||||
Start dummy consumer:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm run dummy-consumer
|
|
||||||
```
|
|
||||||
|
|
||||||
Direct node entrypoint:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
node src/apps/dummy-consumer.mjs
|
|
||||||
```
|
|
||||||
|
|
||||||
## Decision summary
|
|
||||||
|
|
||||||
For an MVP whose job is to answer "what are users asking for right now?", solver websocket quote requests are still the best first source because they are the most direct, timely, and stream-friendly signal. The implementation now routes that signal through Kafka/Redpanda topics so ingestion and downstream reaction can evolve independently.
|
|
||||||
|
|
|
||||||
66
projects/unrip/README.md
Normal file
66
projects/unrip/README.md
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
# unrip project
|
||||||
|
|
||||||
|
This directory contains the trading-system project code and project-specific deployment assets.
|
||||||
|
It is shaped so it can later become its own repository with minimal reshuffling.
|
||||||
|
|
||||||
|
## Contents
|
||||||
|
|
||||||
|
- `src/` — application code
|
||||||
|
- `package.json` / `package-lock.json` — Node package manifest
|
||||||
|
- `Dockerfile` / `.dockerignore` — app container build
|
||||||
|
- `.env.example` — local app runtime example
|
||||||
|
- `compose.yml` — local development stack
|
||||||
|
- `deploy/k8s/base/` — project-specific Kubernetes manifests
|
||||||
|
- `deploy/redpanda/rpk-topics.txt` — project topic reference
|
||||||
|
- `docs/` — project-specific design and contract docs
|
||||||
|
|
||||||
|
## Local development
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd projects/unrip
|
||||||
|
npm install
|
||||||
|
cp .env.example .env
|
||||||
|
# edit .env
|
||||||
|
|
||||||
|
docker compose up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
Useful commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose ps
|
||||||
|
docker compose logs -f
|
||||||
|
docker compose logs -f near-intents-ingest dummy-reactor dummy-executor dummy-consumer
|
||||||
|
npm run near-intents:ingest
|
||||||
|
npm run dummy-reactor
|
||||||
|
npm run dummy-executor
|
||||||
|
npm run dummy-consumer
|
||||||
|
```
|
||||||
|
|
||||||
|
## App image
|
||||||
|
|
||||||
|
The app image is now built from this directory.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd projects/unrip
|
||||||
|
docker build -t unrip:dev .
|
||||||
|
```
|
||||||
|
|
||||||
|
## Kubernetes manifests
|
||||||
|
|
||||||
|
Project manifests live under:
|
||||||
|
|
||||||
|
- `projects/unrip/deploy/k8s/base/`
|
||||||
|
|
||||||
|
They are consumed by the shared Hetzner overlay and bootstrap flow from the repo root.
|
||||||
|
The shared platform remains outside this directory.
|
||||||
|
|
||||||
|
## Shared platform docs
|
||||||
|
|
||||||
|
For cluster/platform/bootstrap details, see the repo-root docs:
|
||||||
|
- `docs/hetzner-k3s-bootstrap.md`
|
||||||
|
- `docs/hetzner-self-hosted-ci-runbook.md`
|
||||||
|
- `docs/k8s-observability.md`
|
||||||
|
- `deploy/k8s/README.md`
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
kind: Kustomization
|
kind: Kustomization
|
||||||
resources:
|
resources:
|
||||||
|
- namespace.yaml
|
||||||
- redpanda.yaml
|
- redpanda.yaml
|
||||||
- unrip.yaml
|
- unrip.yaml
|
||||||
- bootstrap-job.yaml
|
- bootstrap-job.yaml
|
||||||
7
projects/unrip/deploy/k8s/base/namespace.yaml
Normal file
7
projects/unrip/deploy/k8s/base/namespace.yaml
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: unrip
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/part-of: unrip
|
||||||
|
project.pi.io/type: project
|
||||||
85
projects/unrip/docs/contracts.md
Normal file
85
projects/unrip/docs/contracts.md
Normal file
|
|
@ -0,0 +1,85 @@
|
||||||
|
# Event contracts
|
||||||
|
|
||||||
|
## Envelope
|
||||||
|
All bus messages use this envelope:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"event_id": "string",
|
||||||
|
"event_type": "string",
|
||||||
|
"venue": "string",
|
||||||
|
"source": "string|null",
|
||||||
|
"schema_version": 1,
|
||||||
|
"observed_at": "ISO-8601|null",
|
||||||
|
"ingested_at": "ISO-8601",
|
||||||
|
"payload": {},
|
||||||
|
"raw": {}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Topics
|
||||||
|
Current canonical topic set:
|
||||||
|
- `raw.near_intents.quote`
|
||||||
|
- `norm.swap_demand`
|
||||||
|
- `cmd.execute_trade`
|
||||||
|
- `exec.trade_result`
|
||||||
|
|
||||||
|
In Kubernetes bootstrap, Redpanda topic creation is currently handled by the repo-managed bootstrap job applied with the manifest set.
|
||||||
|
|
||||||
|
## `raw.near_intents.quote`
|
||||||
|
- `event_type`: `near_intents_quote_raw`
|
||||||
|
- `payload.message`: original venue-native payload
|
||||||
|
- `raw`: original venue-native payload
|
||||||
|
|
||||||
|
## `norm.swap_demand`
|
||||||
|
- `event_type`: `swap_demand`
|
||||||
|
- payload:
|
||||||
|
- `quote_id`
|
||||||
|
- `asset_in`
|
||||||
|
- `asset_out`
|
||||||
|
- `amount_in`
|
||||||
|
- `amount_out`
|
||||||
|
- `ttl_ms`
|
||||||
|
|
||||||
|
## `cmd.execute_trade`
|
||||||
|
- `event_type`: `execute_trade`
|
||||||
|
- payload:
|
||||||
|
- `command_id`
|
||||||
|
- `idempotency_key`
|
||||||
|
- `execution_key`
|
||||||
|
- `quote_id`
|
||||||
|
- `asset_in`
|
||||||
|
- `asset_out`
|
||||||
|
- `amount_in`
|
||||||
|
- `amount_out`
|
||||||
|
- `reason`
|
||||||
|
|
||||||
|
## `exec.trade_result`
|
||||||
|
- `event_type`: `trade_result`
|
||||||
|
- payload:
|
||||||
|
- `command_id`
|
||||||
|
- `idempotency_key`
|
||||||
|
- `execution_key`
|
||||||
|
- `quote_id`
|
||||||
|
- `status`
|
||||||
|
- `result_code`
|
||||||
|
- `note`
|
||||||
|
|
||||||
|
## Executor idempotency model
|
||||||
|
- `command_id` is unique per trade command and currently deterministic as `cmd-${quote_id}`
|
||||||
|
- `idempotency_key` is stable for semantic duplicate detection and currently `${venue}:${quote_id}`
|
||||||
|
- `execution_key` is the stable partition key and currently `${venue}:${asset_in}->${asset_out}`
|
||||||
|
- executor persists command state on durable storage before publishing a result
|
||||||
|
- already-completed `command_id`s are skipped on replay or restart
|
||||||
|
- if a command is seen again after a persisted `processing` state, the executor emits a recovered result path instead of blindly duplicating work
|
||||||
|
|
||||||
|
## Deployment and persistence implications
|
||||||
|
These contracts are tied to deployment behavior:
|
||||||
|
- executor duplicate suppression depends on durable persistence at `EXECUTOR_STATE_DIR`
|
||||||
|
- local Compose mounts that path for development/runtime testing
|
||||||
|
- the Hetzner single-node k3s path mounts persistent storage for the executor at `/var/lib/unrip/executor-state`
|
||||||
|
- in the current single-node target, that persistence is node-backed and should be treated as required operational state
|
||||||
|
|
||||||
|
Operational consequence:
|
||||||
|
- deleting the executor PVC or losing the node without migration discards idempotency history
|
||||||
|
- that can allow already-seen commands to be treated as new after recovery
|
||||||
198
projects/unrip/docs/minimal-product.md
Normal file
198
projects/unrip/docs/minimal-product.md
Normal file
|
|
@ -0,0 +1,198 @@
|
||||||
|
# Minimal product: NEAR Intents demand monitor
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
Build the smallest useful event-driven product for crypto trading research:
|
||||||
|
|
||||||
|
- read **live user demand** from NEAR Intents
|
||||||
|
- publish demand into a **central Kafka/Redpanda-compatible bus**
|
||||||
|
- prove downstream consumption with a **dummy reactor**
|
||||||
|
- avoid dashboards, execution, wallets, storage, auth workflows beyond the required API key, strategy code, and generic infra beyond the message bus itself
|
||||||
|
|
||||||
|
## Why this is the right first slice
|
||||||
|
From the NEAR Intents docs, there are several possible data surfaces:
|
||||||
|
|
||||||
|
1. **Message Bus WebSocket `quote` subscription**
|
||||||
|
- Endpoint: `wss://solver-relay-v2.chaindefuser.com/ws`
|
||||||
|
- Real-time stream for quote requests
|
||||||
|
- Subscription request shape:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"jsonrpc": "2.0",
|
||||||
|
"id": 1,
|
||||||
|
"method": "subscribe",
|
||||||
|
"params": ["quote"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- Expected live frame shape is JSON-RPC-like but should be treated as flexible. The adapter should accept quote payloads when the useful fields appear either:
|
||||||
|
- directly under `params`
|
||||||
|
- directly under `result`
|
||||||
|
- or at the top level of the message body
|
||||||
|
- Fields of interest include:
|
||||||
|
- `quote_id` (or equivalent request identifier)
|
||||||
|
- `defuse_asset_identifier_in`
|
||||||
|
- `defuse_asset_identifier_out`
|
||||||
|
- `exact_amount_in` or `exact_amount_out`
|
||||||
|
- `min_deadline_ms`
|
||||||
|
- Subscription acknowledgements may also vary. They may arrive as an `id`-matched JSON-RPC response with a simple `result`, a structured `result`, or other non-quote control frame before the first quote event.
|
||||||
|
- This is the closest public signal to **current demand**.
|
||||||
|
|
||||||
|
2. **Message Bus JSON-RPC `publish_intent` / `get_status`**
|
||||||
|
- Endpoint: `https://solver-relay-v2.chaindefuser.com/rpc`
|
||||||
|
- Useful for posting intents or checking a known `intent_hash`
|
||||||
|
- Not a public firehose of all intents.
|
||||||
|
|
||||||
|
3. **Explorer API `/api/v0/transactions`**
|
||||||
|
- Historical and analytics friendly
|
||||||
|
- Requires JWT auth
|
||||||
|
- Better for history, not best for a minimal live monitor
|
||||||
|
|
||||||
|
4. **Verifier contract intent payloads**
|
||||||
|
- The on-chain swap expression is usually `token_diff`
|
||||||
|
- Important for understanding settlement semantics
|
||||||
|
- Not the easiest first live intake path for a lean bus-first system
|
||||||
|
|
||||||
|
## Product decision
|
||||||
|
The minimal product should monitor **WebSocket `quote` events** and route them through a bus-first runtime.
|
||||||
|
|
||||||
|
### Why
|
||||||
|
- closest live signal to user demand
|
||||||
|
- directly reflects what users are requesting from solvers
|
||||||
|
- enough to answer the first trading question: **what assets are being requested right now?**
|
||||||
|
- decouples venue intake from downstream analysis through Kafka-compatible topics
|
||||||
|
|
||||||
|
### Important implementation note
|
||||||
|
Current docs for the market-maker quickstart and live endpoint behavior indicate the Message Bus requires a **partner API key / JWT** in the `Authorization: Bearer ...` header.
|
||||||
|
That means the best path is still the quote stream, but live operation is partner-gated.
|
||||||
|
|
||||||
|
### Important caveat
|
||||||
|
A `quote` event is **pre-trade demand**, not guaranteed execution.
|
||||||
|
That is fine for v0. The purpose is demand sensing, not settlement accounting.
|
||||||
|
|
||||||
|
## Runtime shape
|
||||||
|
|
||||||
|
```text
|
||||||
|
NEAR Intents websocket
|
||||||
|
|
|
||||||
|
v
|
||||||
|
src/apps/near-intents-ingest.mjs
|
||||||
|
|
|
||||||
|
+--> raw.near_intents.quote
|
||||||
|
|
|
||||||
|
+--> norm.swap_demand
|
||||||
|
|
|
||||||
|
v
|
||||||
|
src/apps/dummy-consumer.mjs
|
||||||
|
```
|
||||||
|
|
||||||
|
### Runtime contracts
|
||||||
|
|
||||||
|
#### Ingest app
|
||||||
|
`src/apps/near-intents-ingest.mjs`:
|
||||||
|
- loads env
|
||||||
|
- parses optional `--pair 'asset_a->asset_b'`
|
||||||
|
- starts the NEAR Intents websocket adapter
|
||||||
|
- writes raw and normalized events to the configured broker
|
||||||
|
|
||||||
|
#### Dummy consumer
|
||||||
|
`src/apps/dummy-consumer.mjs`:
|
||||||
|
- subscribes to `norm.swap_demand`
|
||||||
|
- logs observed pair and quote id
|
||||||
|
- exists only to prove a downstream consumer contract
|
||||||
|
|
||||||
|
#### Bus config
|
||||||
|
Default env-driven topics and group ids:
|
||||||
|
- `KAFKA_TOPIC_RAW_NEAR_INTENTS_QUOTE=raw.near_intents.quote`
|
||||||
|
- `KAFKA_TOPIC_NORM_SWAP_DEMAND=norm.swap_demand`
|
||||||
|
- `KAFKA_CONSUMER_GROUP_DUMMY=dummy-reactor-v1`
|
||||||
|
|
||||||
|
Redpanda is a valid runtime target because the transport is Kafka-compatible.
|
||||||
|
|
||||||
|
## Internal model
|
||||||
|
Normalize each quote event into a thin bus envelope:
|
||||||
|
|
||||||
|
Top-level envelope fields:
|
||||||
|
- `venue`
|
||||||
|
- `source`
|
||||||
|
- `type`
|
||||||
|
- `eventId`
|
||||||
|
- `occurredAt`
|
||||||
|
- `ingestedAt`
|
||||||
|
- `assetIn`
|
||||||
|
- `assetOut`
|
||||||
|
- `raw`
|
||||||
|
- `quote`
|
||||||
|
|
||||||
|
Nested `quote` fields:
|
||||||
|
- `quoteId`
|
||||||
|
- `assetIn`
|
||||||
|
- `assetOut`
|
||||||
|
- `amountIn`
|
||||||
|
- `amountOut`
|
||||||
|
- `ttlMs`
|
||||||
|
|
||||||
|
Field extraction must remain tolerant to known upstream aliases, and normalization should continue to operate on the merged `metadata + data` payload shape from the Message Bus event.
|
||||||
|
The live adapter now intentionally accepts quote-like payloads from `params`, `result`, or the top-level message body, but only processes frames that actually look like quote data. Subscription acknowledgements and unrelated control frames should still be ignored.
|
||||||
|
|
||||||
|
## Filtering
|
||||||
|
The ingest runtime supports an optional exact-pair filter:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run near-intents:ingest -- --pair 'asset_a->asset_b'
|
||||||
|
```
|
||||||
|
|
||||||
|
The filter is direction-agnostic, so the reversed asset order is also accepted.
|
||||||
|
|
||||||
|
## Scope boundaries
|
||||||
|
### Must do
|
||||||
|
- connect to the websocket
|
||||||
|
- subscribe to `quote` and tolerate control frames
|
||||||
|
- normalize quote events into one compact model
|
||||||
|
- publish raw and normalized events to Kafka/Redpanda-compatible topics
|
||||||
|
- allow a downstream consumer to react to normalized events
|
||||||
|
- reconnect automatically on disconnect
|
||||||
|
- document `npm` and `node` entrypoints
|
||||||
|
|
||||||
|
### Must not do
|
||||||
|
- Python packaging or CLI guidance
|
||||||
|
- TUI-specific product requirements
|
||||||
|
- charts
|
||||||
|
- account details
|
||||||
|
- pnl
|
||||||
|
- routing internals
|
||||||
|
- market making controls
|
||||||
|
- execution buttons
|
||||||
|
- config panels
|
||||||
|
- speculative infra beyond the current bus and dummy consumer
|
||||||
|
|
||||||
|
## Path to success
|
||||||
|
1. Connect to WebSocket
|
||||||
|
2. Subscribe to `quote`
|
||||||
|
3. Normalize incoming events into one compact model
|
||||||
|
4. Publish raw envelopes to `raw.near_intents.quote`
|
||||||
|
5. Publish normalized envelopes to `norm.swap_demand`
|
||||||
|
6. Start a dummy consumer on the normalized topic
|
||||||
|
7. Reconnect automatically on disconnect
|
||||||
|
8. Only after this works, consider:
|
||||||
|
- `quote_status`-specific downstream handling
|
||||||
|
- historical replay via Explorer API
|
||||||
|
- token metadata enrichment
|
||||||
|
- filtering and alerts beyond `--pair`
|
||||||
|
|
||||||
|
## Packaging alignment
|
||||||
|
Current repository packaging and usage should stay aligned around the JavaScript runtime entrypoints:
|
||||||
|
|
||||||
|
- package scripts:
|
||||||
|
- `npm run near-intents:ingest`
|
||||||
|
- `npm run dummy-consumer`
|
||||||
|
- `npm start` as a compatibility wrapper
|
||||||
|
- direct app entrypoints:
|
||||||
|
- `node src/apps/near-intents-ingest.mjs`
|
||||||
|
- `node src/apps/dummy-consumer.mjs`
|
||||||
|
|
||||||
|
Documentation should treat the npm scripts and `src/apps/*` node entrypoints as canonical. Older single-file and Python/TUI instructions should remain removed to avoid runtime confusion.
|
||||||
|
|
||||||
|
## Sources
|
||||||
|
- NEAR Intents Message Bus WebSocket docs: `subscribe` with `quote` / `quote_status`
|
||||||
|
- NEAR Intents Message Bus RPC docs: `quote`, `publish_intent`, `get_status`
|
||||||
|
- Verifier contract docs: `token_diff` intent type
|
||||||
|
- Explorer API OpenAPI: authenticated historical transactions
|
||||||
383
projects/unrip/docs/next-session-architecture.md
Normal file
383
projects/unrip/docs/next-session-architecture.md
Normal file
|
|
@ -0,0 +1,383 @@
|
||||||
|
# Trading System Architecture Notes for Next Session
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
Build the first real version of the trading system as an event-driven, multi-service architecture.
|
||||||
|
|
||||||
|
Current implemented seed:
|
||||||
|
- NEAR Intents ingest in Node.js
|
||||||
|
- Kafka-compatible bus usage via `kafkajs`
|
||||||
|
- dummy reactor / executor / result consumer loop
|
||||||
|
|
||||||
|
Next session should continue from this architecture, not revert to a monolith, local-only script, or TUI.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Core Architecture
|
||||||
|
All components are independent services.
|
||||||
|
They communicate only through a central Kafka-compatible bus (Redpanda first, Kafka-compatible by design).
|
||||||
|
|
||||||
|
### Service classes
|
||||||
|
- venue ingestors
|
||||||
|
- normalizers
|
||||||
|
- reactors / decision engines
|
||||||
|
- executors
|
||||||
|
- downstream consumers / monitors / archivers / replay tools
|
||||||
|
|
||||||
|
### Service communication rule
|
||||||
|
No direct service-to-service calls for core trading flow.
|
||||||
|
Use bus topics only.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Venue-Oriented Structure
|
||||||
|
The system should be organized by venue.
|
||||||
|
Each venue can have different:
|
||||||
|
- ingest/feed mechanics
|
||||||
|
- normalization logic
|
||||||
|
- execution mechanics
|
||||||
|
|
||||||
|
### Per-venue responsibilities
|
||||||
|
- `ingest` = venue-native intake
|
||||||
|
- `normalize` = convert venue-native payload into canonical internal event
|
||||||
|
- `execute` = venue-specific action logic
|
||||||
|
|
||||||
|
Planned shape:
|
||||||
|
```text
|
||||||
|
src/
|
||||||
|
apps/
|
||||||
|
bus/
|
||||||
|
core/
|
||||||
|
venues/
|
||||||
|
near-intents/
|
||||||
|
ingest
|
||||||
|
normalize
|
||||||
|
execute
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Bus Choice
|
||||||
|
Use **Redpanda** first, but stay fully **Kafka-compatible**.
|
||||||
|
|
||||||
|
### Reason
|
||||||
|
Requirements:
|
||||||
|
- high throughput
|
||||||
|
- low latency
|
||||||
|
- retention
|
||||||
|
- replay
|
||||||
|
- multiple producers/consumers
|
||||||
|
- independent services
|
||||||
|
- future scale-out
|
||||||
|
- multi-language compatibility
|
||||||
|
|
||||||
|
### Constraint
|
||||||
|
Do not use broker-specific features that make migration to Kafka difficult.
|
||||||
|
Use standard Kafka clients and semantics.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Data Model Principles
|
||||||
|
Kafka/Redpanda is the operational event backbone.
|
||||||
|
|
||||||
|
### Event model rules
|
||||||
|
- append-only
|
||||||
|
- immutable events
|
||||||
|
- versioned schemas
|
||||||
|
- raw and normalized events both preserved
|
||||||
|
|
||||||
|
### Every event should include
|
||||||
|
- `event_id`
|
||||||
|
- `event_type`
|
||||||
|
- `venue`
|
||||||
|
- `observed_at` / `ingested_at`
|
||||||
|
- `schema_version`
|
||||||
|
- `payload`
|
||||||
|
- optionally raw/original payload where appropriate
|
||||||
|
|
||||||
|
### Raw vs normalized
|
||||||
|
Keep both.
|
||||||
|
- raw topics = exact venue-native source truth
|
||||||
|
- normalized topics = canonical research/trading inputs
|
||||||
|
|
||||||
|
This is required for:
|
||||||
|
- replay
|
||||||
|
- debugging
|
||||||
|
- future backtesting
|
||||||
|
- future Spark/batch processing
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Current/Planned Topic Flow
|
||||||
|
Minimal 3-stage pipeline:
|
||||||
|
|
||||||
|
1. ingest publishes normalized demand
|
||||||
|
2. reactor publishes trade command
|
||||||
|
3. executor publishes trade result
|
||||||
|
|
||||||
|
### Topic classes
|
||||||
|
- `raw.*` = raw venue-native events
|
||||||
|
- `norm.*` = canonical normalized market events
|
||||||
|
- `cmd.*` = execution commands
|
||||||
|
- `exec.*` = execution outcomes
|
||||||
|
- later `signal.*` if needed for reactor outputs before command stage
|
||||||
|
|
||||||
|
### Current minimal topics
|
||||||
|
- `norm.swap_demand`
|
||||||
|
- `cmd.execute_trade`
|
||||||
|
- `exec.trade_result`
|
||||||
|
|
||||||
|
### NEAR Intents
|
||||||
|
NEAR Intents source currently feeds quote-demand style events from solver-bus websocket.
|
||||||
|
This is a venue ingest source, not the whole trading system.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Execution Safety / Zero Downtime Requirements
|
||||||
|
This is critical.
|
||||||
|
|
||||||
|
### Constraint
|
||||||
|
Multiple executors must never duplicate the same trade/action during deploys, restarts, or rebalances.
|
||||||
|
|
||||||
|
### Must-have rules
|
||||||
|
1. Every execution command must carry a unique `command_id`
|
||||||
|
2. Commands must include deterministic idempotency information
|
||||||
|
3. Executors must be idempotent
|
||||||
|
4. Executors must belong to a consumer group per executor role
|
||||||
|
5. Commands should be partitioned by a stable execution key where ordering matters
|
||||||
|
6. Executor state must be persisted durably enough to detect duplicate command execution
|
||||||
|
|
||||||
|
### Kafka consumer groups are not sufficient alone
|
||||||
|
They help assign work, but they do not guarantee no duplicate processing under restart/rebalance conditions.
|
||||||
|
Idempotency is still required.
|
||||||
|
|
||||||
|
### Rolling updates / zero downtime
|
||||||
|
Executors must support:
|
||||||
|
- graceful shutdown
|
||||||
|
- stop taking new work before exit
|
||||||
|
- finish or safely recover in-flight work
|
||||||
|
- commit offsets only after safe execution state transition
|
||||||
|
|
||||||
|
### Persistence implication
|
||||||
|
Executor idempotency state is not optional metadata.
|
||||||
|
It is operational state that must survive pod restarts.
|
||||||
|
|
||||||
|
Current single-node k3s direction:
|
||||||
|
- executor state lives at `/var/lib/unrip/executor-state`
|
||||||
|
- Kubernetes mounts that path through persistent storage
|
||||||
|
- the Hetzner single-node overlay currently targets k3s `local-path` storage
|
||||||
|
- node loss without storage migration means duplicate-suppression history is lost
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Deployment Target
|
||||||
|
### First deployment phase
|
||||||
|
- single machine on Hetzner
|
||||||
|
- but still multiple independent services
|
||||||
|
- no architecture shortcuts that prevent future clustering
|
||||||
|
|
||||||
|
### Future target
|
||||||
|
- split across multiple machines
|
||||||
|
- cluster capable
|
||||||
|
- fault tolerant
|
||||||
|
- multi-node
|
||||||
|
- zero-downtime deploys
|
||||||
|
|
||||||
|
### Deployment rules from day 1
|
||||||
|
- every component is a separate container/service
|
||||||
|
- all config via env/config files
|
||||||
|
- communication over network/bus only
|
||||||
|
- persistent components use mounted volumes/PVCs
|
||||||
|
- no manual SSH-based operational workflow
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Infrastructure / Ops Direction
|
||||||
|
Target environment:
|
||||||
|
- Hetzner
|
||||||
|
- self-hosted CI/CD
|
||||||
|
- provisioning by code
|
||||||
|
- no GitHub dependency
|
||||||
|
|
||||||
|
### Desired stack direction
|
||||||
|
- Terraform for Hetzner provisioning
|
||||||
|
- Kubernetes-oriented target from the start
|
||||||
|
- self-hosted Git + CI/CD
|
||||||
|
- Kafka-compatible broker
|
||||||
|
- object storage later for long-term archived event history
|
||||||
|
|
||||||
|
### Single-node first, future cluster later
|
||||||
|
The first version may run on one machine, but deployment structure should already match a future distributed system.
|
||||||
|
|
||||||
|
### Current canonical operator path
|
||||||
|
The repo now documents and partially implements this path as the primary deployment workflow:
|
||||||
|
|
||||||
|
#### Phase 0: workstation bootstrap
|
||||||
|
1. A local operator workstation prepares bootstrap secrets in `scripts/hetzner/bootstrap-secrets.env`.
|
||||||
|
2. The operator runs `bash scripts/hetzner/bootstrap.sh`.
|
||||||
|
3. Terraform provisions the server, firewall, network, and cloud-init user-data.
|
||||||
|
4. cloud-init installs k3s automatically and prepares persistence directories plus bootstrap artifacts.
|
||||||
|
5. The workstation waits for the public k3s API endpoint to report ready.
|
||||||
|
6. The workstation writes `.state/hetzner/kubeconfig.yaml`.
|
||||||
|
7. The workstation injects initial Kubernetes Secrets for app and Forgejo bootstrap.
|
||||||
|
8. The workstation applies repo-managed Kubernetes manifests under `deploy/k8s/`.
|
||||||
|
9. The workstation performs the first image/bootstrap delivery attempt for the app workloads.
|
||||||
|
10. The workstation verifies rollout status.
|
||||||
|
|
||||||
|
#### Phase 1: self-hosted handoff
|
||||||
|
1. Forgejo becomes reachable in-cluster.
|
||||||
|
2. The operator completes initial Forgejo admin/repo setup.
|
||||||
|
3. This repo is pushed or mirrored into Forgejo.
|
||||||
|
4. The Forgejo runner becomes the routine app deployment mechanism.
|
||||||
|
5. Terraform remains the infra mutation entrypoint unless further automated later.
|
||||||
|
|
||||||
|
### Failure-recovery expectation
|
||||||
|
The bootstrap path must be rerunnable from the workstation.
|
||||||
|
Docs should keep treating recovery as:
|
||||||
|
- fix local secrets/inputs
|
||||||
|
- rerun the bootstrap script
|
||||||
|
- inspect the cluster with the generated kubeconfig
|
||||||
|
- destroy/recreate infra with `scripts/hetzner/destroy.sh` only when required
|
||||||
|
|
||||||
|
### Current repo-state caveats
|
||||||
|
The direction is clear, but the implementation is still mid-transition:
|
||||||
|
- the bootstrap script currently applies `deploy/k8s/base` directly rather than the Hetzner overlay
|
||||||
|
- kubeconfig/auth handling is not yet fully production-hardened
|
||||||
|
- first image delivery is still a bootstrap workaround rather than a final registry-native CI path
|
||||||
|
- Forgejo admin bootstrap, repo creation, and Actions configuration still require operator steps
|
||||||
|
- local Compose remains in the repo for development/testing, not as the canonical production path
|
||||||
|
|
||||||
|
### Minimal repo layout target
|
||||||
|
```text
|
||||||
|
deploy/
|
||||||
|
hetzner/
|
||||||
|
README.md
|
||||||
|
k8s/
|
||||||
|
base/
|
||||||
|
overlays/
|
||||||
|
hetzner-single-node/
|
||||||
|
infra/
|
||||||
|
terraform/
|
||||||
|
hetzner/
|
||||||
|
```
|
||||||
|
|
||||||
|
Guidelines:
|
||||||
|
- `infra/terraform/hetzner/` owns VM, firewall, networking, and cloud-init rendering
|
||||||
|
- `deploy/k8s/` owns Kubernetes-native manifests and overlays
|
||||||
|
- app runtime manifests should remain Kubernetes-native so they can later move from single-node k3s to a larger cluster with minimal rewrite
|
||||||
|
- secret material must not live in git in plaintext; bootstrap docs should describe workstation-driven injection or generated secret references
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Local Development / Testing Direction
|
||||||
|
Do not assume manual multi-terminal operation long term.
|
||||||
|
|
||||||
|
### Requirement
|
||||||
|
Need an orchestrated local/dev runtime.
|
||||||
|
|
||||||
|
### Local dev should preserve real boundaries
|
||||||
|
- separate services
|
||||||
|
- broker present
|
||||||
|
- env/config driven
|
||||||
|
- same event flow as production
|
||||||
|
|
||||||
|
### Current local/dev answer
|
||||||
|
Compose is still acceptable for:
|
||||||
|
- developer laptops
|
||||||
|
- fast local iteration
|
||||||
|
- debugging event flow
|
||||||
|
- validating container boundaries before Kubernetes rollout
|
||||||
|
|
||||||
|
But Compose should remain explicitly secondary to the repo-driven Hetzner + k3s path for production operations.
|
||||||
|
|
||||||
|
### Testing layers
|
||||||
|
1. unit tests for normalizers / schema logic / helpers
|
||||||
|
2. integration tests against Kafka-compatible broker
|
||||||
|
3. replay/simulation tests using retained event streams
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Spark Readiness
|
||||||
|
Do not add Spark now.
|
||||||
|
But keep the system Spark-compatible later by:
|
||||||
|
- preserving raw events
|
||||||
|
- preserving normalized events
|
||||||
|
- using immutable append-only event streams
|
||||||
|
- versioning schemas
|
||||||
|
- separating operational event log from future analytical processing
|
||||||
|
|
||||||
|
Spark later would be for:
|
||||||
|
- large-scale backtesting
|
||||||
|
- feature generation
|
||||||
|
- archive processing
|
||||||
|
- multi-venue analytics
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Immediate Next Engineering Tasks
|
||||||
|
Next session should focus on the following.
|
||||||
|
|
||||||
|
### 1. Clean current repo structure
|
||||||
|
Remove duplicate/legacy paths and keep one canonical structure only.
|
||||||
|
|
||||||
|
### 2. Keep/complete the 3-stage loop
|
||||||
|
- NEAR Intents ingest -> `norm.swap_demand`
|
||||||
|
- dummy reactor -> `cmd.execute_trade`
|
||||||
|
- dummy executor -> `exec.trade_result`
|
||||||
|
- downstream result consumer
|
||||||
|
|
||||||
|
### 3. Define canonical schemas
|
||||||
|
Define concrete event schemas for:
|
||||||
|
- normalized swap demand
|
||||||
|
- execute trade command
|
||||||
|
- trade result
|
||||||
|
|
||||||
|
### 4. Define executor idempotency model
|
||||||
|
Specify:
|
||||||
|
- `command_id`
|
||||||
|
- idempotency key rules
|
||||||
|
- execution state transition rules
|
||||||
|
- duplicate handling rules
|
||||||
|
|
||||||
|
### 5. Move toward production-shaped deployment
|
||||||
|
Design for:
|
||||||
|
- one service per container
|
||||||
|
- single-node deployment first
|
||||||
|
- future multi-node split without app rewrite
|
||||||
|
|
||||||
|
### 6. Harden provisioning/deployment path
|
||||||
|
Next infra work should continue improving:
|
||||||
|
- Hetzner provisioning by code
|
||||||
|
- workstation bootstrap rerunnability
|
||||||
|
- self-hosted CI/CD handoff
|
||||||
|
- registry-native image delivery
|
||||||
|
- overlay convergence for the Hetzner single-node target
|
||||||
|
|
||||||
|
Status update:
|
||||||
|
- minimal Terraform exists under `infra/terraform/hetzner`
|
||||||
|
- first boot is cloud-init driven and installs k3s automatically
|
||||||
|
- bootstrap now starts from a local operator workstation rather than manual host login
|
||||||
|
- Kubernetes assets exist under `deploy/k8s`
|
||||||
|
- executor persistence boundaries are explicit for single-node k3s
|
||||||
|
- self-hosted CI handoff is documented, but still requires follow-up hardening
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Non-Goals for Next Session
|
||||||
|
- no dashboards
|
||||||
|
- no UI/TUI
|
||||||
|
- no monolith convenience architecture
|
||||||
|
- no SQLite-first system of record
|
||||||
|
- no direct coupling between ingest, decision, and execution
|
||||||
|
- no temporary local-only shortcuts that block future cluster deployment
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Guiding Principle
|
||||||
|
Build the single-node first version as if it is already a distributed system:
|
||||||
|
- separate services
|
||||||
|
- durable event bus
|
||||||
|
- replayable events
|
||||||
|
- explicit contracts
|
||||||
|
- idempotent execution
|
||||||
|
- production-compatible deployment boundaries
|
||||||
|
- bootstrapable from scratch without manual SSH-based host setup
|
||||||
144
projects/unrip/docs/spec.md
Normal file
144
projects/unrip/docs/spec.md
Normal file
|
|
@ -0,0 +1,144 @@
|
||||||
|
# NEAR Intents demand monitor: bus-first source plan
|
||||||
|
|
||||||
|
## Why websocket quote requests are still the MVP demand signal
|
||||||
|
|
||||||
|
Public solver quote requests remain the closest thing to live demand because they appear when a user or integration asks the network for executable pricing. They are still the right upstream source, but the runtime architecture is now bus-first rather than terminal-first.
|
||||||
|
|
||||||
|
Why this source wins for a first monitor:
|
||||||
|
|
||||||
|
- **Most real-time:** quote requests arrive before settlement and usually before a completed trade is visible anywhere else.
|
||||||
|
- **Closer to intent formation:** they reflect active user demand, not just historical outcomes.
|
||||||
|
- **Operationally simple:** a single websocket feed can drive the ingest side without indexing chains, scraping dashboards, or correlating multiple APIs.
|
||||||
|
- **Good enough for ranking demand:** even if quotes do not always become fills, repeated quote flow is still a strong indicator of what users are currently trying to do.
|
||||||
|
|
||||||
|
## Tradeoffs vs other sources
|
||||||
|
|
||||||
|
### Solver websocket quote requests
|
||||||
|
|
||||||
|
Pros:
|
||||||
|
- lowest-latency view of current demand
|
||||||
|
- directly tied to solver workflow
|
||||||
|
- suitable for a streaming ingest adapter
|
||||||
|
- can be normalized into pair, size, and frequency metrics immediately
|
||||||
|
|
||||||
|
Cons:
|
||||||
|
- quote requests are **interest**, not guaranteed executed volume
|
||||||
|
- public access may still be rate-limited, undocumented, or require credentials depending on environment
|
||||||
|
- schema and availability may change faster than user-facing products
|
||||||
|
|
||||||
|
### Explorer
|
||||||
|
|
||||||
|
Explorer (`https://explorer.near-intents.org/`) is useful for validation and historical inspection, but it is usually a worse primary source for an MVP demand monitor.
|
||||||
|
|
||||||
|
Tradeoffs:
|
||||||
|
- better for human inspection than low-latency streaming
|
||||||
|
- likely shows processed/published activity instead of raw quote demand
|
||||||
|
- may lag the actual request path
|
||||||
|
- less convenient as a machine-first demand feed
|
||||||
|
|
||||||
|
### Status dashboard / published status
|
||||||
|
|
||||||
|
Status (`https://status.near-intents.org/posts/dashboard`) is useful for system health, not demand discovery.
|
||||||
|
|
||||||
|
Tradeoffs:
|
||||||
|
- tells us whether the platform is up, degraded, or incident-affected
|
||||||
|
- does **not** represent per-request user demand
|
||||||
|
- coarse and aggregated by design
|
||||||
|
|
||||||
|
### Published intents / settled outcomes
|
||||||
|
|
||||||
|
Published or completed intents are higher-confidence signals, but lower-fidelity for immediate demand sensing.
|
||||||
|
|
||||||
|
Tradeoffs:
|
||||||
|
- stronger evidence of actual execution
|
||||||
|
- misses abandoned demand and pre-trade discovery
|
||||||
|
- arrives later than quote traffic
|
||||||
|
- may require more indexing and entity correlation work
|
||||||
|
|
||||||
|
## Runtime architecture
|
||||||
|
|
||||||
|
```text
|
||||||
|
solver websocket quote stream
|
||||||
|
|
|
||||||
|
v
|
||||||
|
src/apps/near-intents-ingest.mjs
|
||||||
|
|
|
||||||
|
+--> raw.near_intents.quote
|
||||||
|
|
|
||||||
|
+--> norm.swap_demand
|
||||||
|
|
|
||||||
|
v
|
||||||
|
src/apps/dummy-consumer.mjs
|
||||||
|
```
|
||||||
|
|
||||||
|
### Responsibilities
|
||||||
|
|
||||||
|
#### `src/apps/near-intents-ingest.mjs`
|
||||||
|
- loads env from `.env`
|
||||||
|
- parses optional `--pair 'asset_a->asset_b'`
|
||||||
|
- connects to the NEAR Intents websocket
|
||||||
|
- subscribes to `quote` and `quote_status`
|
||||||
|
- publishes raw venue envelopes to `raw.near_intents.quote`
|
||||||
|
- publishes normalized swap-demand envelopes to `norm.swap_demand`
|
||||||
|
|
||||||
|
#### `src/apps/dummy-consumer.mjs`
|
||||||
|
- consumes normalized events from `norm.swap_demand`
|
||||||
|
- logs observed demand as a placeholder for later strategy logic
|
||||||
|
|
||||||
|
#### Kafka / Redpanda layer
|
||||||
|
- broker endpoint comes from `KAFKA_BROKERS`
|
||||||
|
- Redpanda is supported through Kafka protocol compatibility
|
||||||
|
- topics are configurable via env and default to:
|
||||||
|
- `raw.near_intents.quote`
|
||||||
|
- `norm.swap_demand`
|
||||||
|
|
||||||
|
## Assumptions and limitations
|
||||||
|
|
||||||
|
- The websocket is the best available **MVP** source, not a perfect truth source.
|
||||||
|
- Demand is approximated by quote requests, not by settled intents.
|
||||||
|
- Live endpoints require auth in practice; `NEAR_INTENTS_API_KEY` must be provided.
|
||||||
|
- Request schemas may evolve; the parser should tolerate missing fields.
|
||||||
|
- The current product is intentionally minimal: no database, no backfill, no reconciliation against chain state.
|
||||||
|
- The dummy consumer proves the decoupled flow but is not a strategy engine.
|
||||||
|
|
||||||
|
## Run instructions
|
||||||
|
|
||||||
|
Install:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm install
|
||||||
|
```
|
||||||
|
|
||||||
|
Start ingest:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run near-intents:ingest
|
||||||
|
```
|
||||||
|
|
||||||
|
Direct node entrypoint:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
node src/apps/near-intents-ingest.mjs
|
||||||
|
```
|
||||||
|
|
||||||
|
Run with exact-pair filtering:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run near-intents:ingest -- --pair 'asset_a->asset_b'
|
||||||
|
```
|
||||||
|
|
||||||
|
Start dummy consumer:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run dummy-consumer
|
||||||
|
```
|
||||||
|
|
||||||
|
Direct node entrypoint:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
node src/apps/dummy-consumer.mjs
|
||||||
|
```
|
||||||
|
|
||||||
|
## Decision summary
|
||||||
|
|
||||||
|
For an MVP whose job is to answer "what are users asking for right now?", solver websocket quote requests are still the best first source because they are the most direct, timely, and stream-friendly signal. The implementation now routes that signal through Kafka/Redpanda topics so ingestion and downstream reaction can evolve independently.
|
||||||
|
|
@ -34,7 +34,8 @@ export SSH_PUBLIC_KEY_PATH="${SSH_PUBLIC_KEY_PATH:-$HOME/.ssh/id_ed25519.pub}"
|
||||||
export PROJECT_NAME="${PROJECT_NAME:-unrip}"
|
export PROJECT_NAME="${PROJECT_NAME:-unrip}"
|
||||||
export PROJECT_NAMESPACE="${PROJECT_NAMESPACE:-$PROJECT_NAME}"
|
export PROJECT_NAMESPACE="${PROJECT_NAMESPACE:-$PROJECT_NAME}"
|
||||||
# export PROJECT_OVERLAY_DIR="$PWD/deploy/k8s/overlays/hetzner-single-node"
|
# export PROJECT_OVERLAY_DIR="$PWD/deploy/k8s/overlays/hetzner-single-node"
|
||||||
# export PROJECT_KUSTOMIZE_PATH="../../projects/unrip/base"
|
# export PROJECT_DIR="$PWD/projects/unrip"
|
||||||
|
# export PROJECT_KUSTOMIZE_PATH="../../../../projects/unrip/deploy/k8s/base"
|
||||||
# export PROJECT_SECRET_NAME="unrip-secrets"
|
# export PROJECT_SECRET_NAME="unrip-secrets"
|
||||||
# export PROJECT_SECRET_ENV_BASENAME="unrip.env"
|
# export PROJECT_SECRET_ENV_BASENAME="unrip.env"
|
||||||
# export PROJECT_REGISTRY_SECRET_NAME="unrip-registry-creds"
|
# export PROJECT_REGISTRY_SECRET_NAME="unrip-registry-creds"
|
||||||
|
|
|
||||||
|
|
@ -56,9 +56,11 @@ resolve_secret_var PORKBUN_SECRET_API_KEY optional
|
||||||
: "${PROJECT_NAME:=$DEFAULT_PROJECT_NAME}"
|
: "${PROJECT_NAME:=$DEFAULT_PROJECT_NAME}"
|
||||||
: "${PROJECT_NAMESPACE:=$DEFAULT_PROJECT_NAMESPACE}"
|
: "${PROJECT_NAMESPACE:=$DEFAULT_PROJECT_NAMESPACE}"
|
||||||
: "${PROJECT_OVERLAY_DIR:=$OVERLAY_DIR}"
|
: "${PROJECT_OVERLAY_DIR:=$OVERLAY_DIR}"
|
||||||
|
: "${PROJECT_DIR:=$ROOT_DIR/projects/${PROJECT_NAME}}"
|
||||||
|
: "${PROJECT_REPO_PATH:=$(realpath --relative-to="$ROOT_DIR" "$PROJECT_DIR")}"
|
||||||
: "${BOOTSTRAP_NODE_NAME:=unrip-1}"
|
: "${BOOTSTRAP_NODE_NAME:=unrip-1}"
|
||||||
: "${SKIP_TERRAFORM_APPLY:=0}"
|
: "${SKIP_TERRAFORM_APPLY:=0}"
|
||||||
: "${PROJECT_KUSTOMIZE_PATH:=../../projects/${PROJECT_NAME}/base}"
|
: "${PROJECT_KUSTOMIZE_PATH:=../../../../projects/${PROJECT_NAME}/deploy/k8s/base}"
|
||||||
: "${PROJECT_SECRET_NAME:=${PROJECT_NAME}-secrets}"
|
: "${PROJECT_SECRET_NAME:=${PROJECT_NAME}-secrets}"
|
||||||
: "${PROJECT_SECRET_ENV_BASENAME:=${PROJECT_NAME}.env}"
|
: "${PROJECT_SECRET_ENV_BASENAME:=${PROJECT_NAME}.env}"
|
||||||
: "${PROJECT_REGISTRY_SECRET_NAME:=${PROJECT_NAME}-registry-creds}"
|
: "${PROJECT_REGISTRY_SECRET_NAME:=${PROJECT_NAME}-registry-creds}"
|
||||||
|
|
@ -492,6 +494,7 @@ if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then
|
||||||
--project-name "$PROJECT_NAME"
|
--project-name "$PROJECT_NAME"
|
||||||
--project-namespace "$PROJECT_NAMESPACE"
|
--project-namespace "$PROJECT_NAMESPACE"
|
||||||
--project-deployments "${PROJECT_DEPLOYMENTS// /,}"
|
--project-deployments "${PROJECT_DEPLOYMENTS// /,}"
|
||||||
|
--project-path "$PROJECT_REPO_PATH"
|
||||||
)
|
)
|
||||||
if [[ "$FORGEJO_REPO_PRIVATE" == "true" ]]; then
|
if [[ "$FORGEJO_REPO_PRIVATE" == "true" ]]; then
|
||||||
forgejo_bootstrap_args+=(--repo-private)
|
forgejo_bootstrap_args+=(--repo-private)
|
||||||
|
|
@ -503,7 +506,7 @@ if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then
|
||||||
wait_for_url "$FORGEJO_ROOT_URL" "Forgejo public URL" 180 5
|
wait_for_url "$FORGEJO_ROOT_URL" "Forgejo public URL" 180 5
|
||||||
wait_for_http_status "https://$REGISTRY_DOMAIN/v2/" "registry public URL" '200|401' 180 5
|
wait_for_http_status "https://$REGISTRY_DOMAIN/v2/" "registry public URL" '200|401' 180 5
|
||||||
else
|
else
|
||||||
docker build -t "$BOOTSTRAP_IMAGE" "$ROOT_DIR"
|
docker build -t "$BOOTSTRAP_IMAGE" "$PROJECT_DIR"
|
||||||
docker save "$BOOTSTRAP_IMAGE" \
|
docker save "$BOOTSTRAP_IMAGE" \
|
||||||
| ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$SSH_TARGET" 'sudo k3s ctr images import -'
|
| ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$SSH_TARGET" 'sudo k3s ctr images import -'
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -113,6 +113,7 @@ def main():
|
||||||
parser.add_argument('--project-name', required=True)
|
parser.add_argument('--project-name', required=True)
|
||||||
parser.add_argument('--project-namespace', required=True)
|
parser.add_argument('--project-namespace', required=True)
|
||||||
parser.add_argument('--project-deployments', required=True)
|
parser.add_argument('--project-deployments', required=True)
|
||||||
|
parser.add_argument('--project-path', required=True)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
client = ForgejoClient(args.forgejo_url, args.admin_username, args.admin_password, args.token)
|
client = ForgejoClient(args.forgejo_url, args.admin_username, args.admin_password, args.token)
|
||||||
|
|
@ -133,6 +134,7 @@ def main():
|
||||||
client.upsert_variable(args.repo_owner, args.repo_name, 'PROJECT_NAME', args.project_name)
|
client.upsert_variable(args.repo_owner, args.repo_name, 'PROJECT_NAME', args.project_name)
|
||||||
client.upsert_variable(args.repo_owner, args.repo_name, 'PROJECT_NAMESPACE', args.project_namespace)
|
client.upsert_variable(args.repo_owner, args.repo_name, 'PROJECT_NAMESPACE', args.project_namespace)
|
||||||
client.upsert_variable(args.repo_owner, args.repo_name, 'PROJECT_DEPLOYMENTS', args.project_deployments)
|
client.upsert_variable(args.repo_owner, args.repo_name, 'PROJECT_DEPLOYMENTS', args.project_deployments)
|
||||||
|
client.upsert_variable(args.repo_owner, args.repo_name, 'PROJECT_PATH', args.project_path)
|
||||||
print('upserted repo action variables')
|
print('upserted repo action variables')
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue