#!/usr/bin/env bash set -euo pipefail ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd) # shellcheck disable=SC1091 source "$ROOT_DIR/scripts/hetzner/lib.sh" load_bootstrap_env TF_DIR="$ROOT_DIR/infra/terraform/hetzner" STATE_DIR="$ROOT_DIR/.state/hetzner" KUBECONFIG_PATH="$STATE_DIR/kubeconfig.yaml" CI_KUBECONFIG_PATH="$STATE_DIR/kubeconfig.incluster.yaml" OVERLAY_DIR="$ROOT_DIR/deploy/k8s/overlays/hetzner-single-node" DEFAULT_PROJECT_NAME="unrip" DEFAULT_PROJECT_NAMESPACE="$DEFAULT_PROJECT_NAME" mkdir -p "$STATE_DIR" require terraform require kubectl require curl require python3 require ssh require git require base64 require realpath require_python_modules python3 yaml resolve_secret_var HCLOUD_TOKEN required resolve_secret_var TAILSCALE_AUTH_KEY optional resolve_secret_var NEAR_INTENTS_API_KEY required resolve_secret_var REGISTRY_PASSWORD required resolve_secret_var FORGEJO_ADMIN_PASSWORD required resolve_secret_var GRAFANA_ADMIN_PASSWORD optional resolve_secret_var CLOUDFLARE_API_TOKEN optional resolve_secret_var CLOUDFLARE_ZONE_ID optional resolve_secret_var PORKBUN_API_KEY optional resolve_secret_var PORKBUN_SECRET_API_KEY optional : "${SSH_PUBLIC_KEY_PATH:?set SSH_PUBLIC_KEY_PATH}" : "${PUBLIC_DOMAIN:?set PUBLIC_DOMAIN}" : "${LETSENCRYPT_EMAIL:?set LETSENCRYPT_EMAIL}" : "${BASE_DOMAIN:?set BASE_DOMAIN}" : "${FORGEJO_DOMAIN:=git.${PUBLIC_DOMAIN}}" : "${FORGEJO_ROOT_URL:=https://${FORGEJO_DOMAIN}/}" : "${FORGEJO_INTERNAL_URL:=http://forgejo.forgejo.svc.cluster.local:3000/}" : "${REGISTRY_DOMAIN:=registry.${PUBLIC_DOMAIN}}" : "${GRAFANA_DOMAIN:=grafana.${PUBLIC_DOMAIN}}" : "${GRAFANA_ROOT_URL:=https://${GRAFANA_DOMAIN}/}" : "${HEADLAMP_DOMAIN:=headlamp.${PUBLIC_DOMAIN}}" : "${GRAFANA_ADMIN_USERNAME:=admin}" : "${REGISTRY_USERNAME:?set REGISTRY_USERNAME}" : "${TAILSCALE_CONTROL_PLANE_HOSTNAME:=}" : "${TF_ADMIN_CIDR_BLOCKS:=}" : "${BOOTSTRAP_ALLOW_PUBLIC_ADMIN_FALLBACK:=1}" : "${PROJECT_NAME:=$DEFAULT_PROJECT_NAME}" : "${PROJECT_NAMESPACE:=$DEFAULT_PROJECT_NAMESPACE}" : "${PROJECT_OVERLAY_DIR:=$OVERLAY_DIR}" : "${PROJECT_DIR:=$ROOT_DIR}" : "${PROJECT_REPO_PATH:=.}" : "${BOOTSTRAP_NODE_NAME:=unrip-1}" : "${SKIP_TERRAFORM_APPLY:=0}" : "${PROJECT_KUSTOMIZE_PATH:=}" : "${PROJECT_SECRET_NAME:=${PROJECT_NAME}-secrets}" : "${PROJECT_SECRET_ENV_BASENAME:=${PROJECT_NAME}.env}" : "${PROJECT_REGISTRY_SECRET_NAME:=${PROJECT_NAME}-registry-creds}" : "${PROJECT_IMAGE_REPOSITORY:=${PROJECT_NAME}}" : "${PROJECT_DEPLOYMENTS:=near-intents-ingest dummy-reactor dummy-executor dummy-consumer}" : "${FORGEJO_ADMIN_USERNAME:?set FORGEJO_ADMIN_USERNAME}" : "${FORGEJO_ADMIN_EMAIL:?set FORGEJO_ADMIN_EMAIL}" : "${FORGEJO_RUNNER_NAME:=k3s-runner}" : "${FORGEJO_RUNNER_LABELS:=linux-amd64:host}" : "${FORGEJO_REPO_OWNER:=$FORGEJO_ADMIN_USERNAME}" : "${FORGEJO_REPO_NAME:=$(basename "$ROOT_DIR")}" : "${FORGEJO_REPO_PRIVATE:=true}" : "${BOOTSTRAP_DELIVERY_MODE:=forgejo-actions}" BOOTSTRAP_IMAGE="${PROJECT_IMAGE_REPOSITORY}:bootstrap" PROJECT_SECRET_ENV_PATH="$PROJECT_OVERLAY_DIR/secrets/$PROJECT_SECRET_ENV_BASENAME" GENERATED_OVERLAY_DIR="$STATE_DIR/generated-overlay" if [[ "$BOOTSTRAP_DELIVERY_MODE" != "forgejo-actions" ]]; then require docker fi if [[ -n "${TAILSCALE_AUTH_KEY:-}" && "$TF_ADMIN_CIDR_BLOCKS" == '[]' && "$BOOTSTRAP_ALLOW_PUBLIC_ADMIN_FALLBACK" == "1" ]]; then OPERATOR_PUBLIC_IP="$(curl -fsS https://api.ipify.org || true)" if [[ "$OPERATOR_PUBLIC_IP" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then TF_ADMIN_CIDR_BLOCKS="[\"${OPERATOR_PUBLIC_IP}/32\"]" echo "tailscale bootstrap fallback enabled for operator public IP ${OPERATOR_PUBLIC_IP}/32" fi fi SSH_PUBLIC_KEY=$(cat "$SSH_PUBLIC_KEY_PATH") SSH_PRIVATE_KEY_PATH="${SSH_PUBLIC_KEY_PATH%.pub}" if [[ ! -f "$SSH_PRIVATE_KEY_PATH" ]]; then echo "missing ssh private key for bootstrap: $SSH_PRIVATE_KEY_PATH" >&2 exit 1 fi TF_VARS=( -var "hcloud_token=$HCLOUD_TOKEN" -var "ssh_public_key=$SSH_PUBLIC_KEY" -var "public_domain=$PUBLIC_DOMAIN" -var "tailscale_auth_key=${TAILSCALE_AUTH_KEY:-}" -var "tailscale_control_plane_hostname=${TAILSCALE_CONTROL_PLANE_HOSTNAME:-}" ) if [[ -n "$TF_ADMIN_CIDR_BLOCKS" && "$TF_ADMIN_CIDR_BLOCKS" != '[]' ]]; then TF_VARS+=(-var "admin_cidr_blocks=$TF_ADMIN_CIDR_BLOCKS") fi if [[ -n "${TAILSCALE_AUTH_KEY:-}" ]]; then bash "$ROOT_DIR/scripts/hetzner/print-tailscale-firewall-note.sh" fi terraform -chdir="$TF_DIR" init if [[ "$SKIP_TERRAFORM_APPLY" != "1" ]]; then terraform -chdir="$TF_DIR" apply -auto-approve "${TF_VARS[@]}" fi SERVER_IP=$(terraform -chdir="$TF_DIR" output -raw server_ipv4) K3S_API_URL=$(terraform -chdir="$TF_DIR" output -raw k3s_api_url) SSH_TARGET="root@${SERVER_IP}" USE_SSH_TUNNEL_FOR_K3S=0 if [[ -n "${TAILSCALE_AUTH_KEY:-}" ]]; then DISCOVERED_TAILSCALE_HOST="$(wait_for_tailscale_node "$BOOTSTRAP_NODE_NAME" 24 5 || true)" if [[ -n "$DISCOVERED_TAILSCALE_HOST" ]]; then SSH_TARGET="root@${DISCOVERED_TAILSCALE_HOST}" USE_SSH_TUNNEL_FOR_K3S=1 elif [[ "$TF_ADMIN_CIDR_BLOCKS" != '[]' ]]; then echo "tailscale node did not appear in time; falling back to public admin access for this bootstrap run" >&2 else echo "tailscale node did not appear in time and no public admin fallback is configured" >&2 exit 1 fi fi if [[ -n "${CLOUDFLARE_API_TOKEN:-}" && -n "${CLOUDFLARE_ZONE_ID:-}" ]]; then if ! SERVER_IP="$SERVER_IP" BASE_DOMAIN="$BASE_DOMAIN" bash "$ROOT_DIR/scripts/hetzner/configure-cloudflare-dns.sh"; then echo "warning: cloudflare DNS automation failed; continuing without automated DNS" >&2 fi elif [[ -n "${PORKBUN_API_KEY:-}" && -n "${PORKBUN_SECRET_API_KEY:-}" ]]; then if ! SERVER_IP="$SERVER_IP" BASE_DOMAIN="$BASE_DOMAIN" bash "$ROOT_DIR/scripts/hetzner/configure-porkbun-dns.sh"; then echo "warning: porkbun DNS automation failed; continuing without automated DNS" >&2 fi fi wait_for_ssh "$SSH_TARGET" if [[ "$USE_SSH_TUNNEL_FOR_K3S" == "1" ]]; then LOCAL_K3S_TUNNEL_PORT="${LOCAL_K3S_TUNNEL_PORT:-$(python3 - <<'PY' import socket s=socket.socket() s.bind(('127.0.0.1', 0)) print(s.getsockname()[1]) s.close() PY )}" K3S_API_URL="https://localhost:${LOCAL_K3S_TUNNEL_PORT}" ssh -i "$SSH_PRIVATE_KEY_PATH" \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -o ExitOnForwardFailure=yes \ -o ServerAliveInterval=30 \ -o ServerAliveCountMax=3 \ -N -L "127.0.0.1:${LOCAL_K3S_TUNNEL_PORT}:127.0.0.1:6443" \ "$SSH_TARGET" >/dev/null 2>&1 & K3S_TUNNEL_PID=$! trap 'if [[ -n "${K3S_TUNNEL_PID:-}" ]]; then kill "$K3S_TUNNEL_PID" >/dev/null 2>&1 || true; fi' EXIT fi echo "waiting for Kubernetes API on $K3S_API_URL..." ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$SSH_TARGET" 'sudo cat /etc/rancher/k3s/k3s.yaml' \ | sed "s|https://127.0.0.1:6443|${K3S_API_URL}|" > "$KUBECONFIG_PATH" export KUBECONFIG="$KUBECONFIG_PATH" wait_for_kubectl 120 5 python3 - "$KUBECONFIG_PATH" "$CI_KUBECONFIG_PATH" <<'PY' import sys import yaml src, dst = sys.argv[1], sys.argv[2] config = yaml.safe_load(open(src)) config['clusters'][0]['cluster']['server'] = 'https://kubernetes.default.svc:443' yaml.safe_dump(config, open(dst, 'w'), sort_keys=False) PY mkdir -p "$PROJECT_OVERLAY_DIR/secrets" "$GENERATED_OVERLAY_DIR" OBSERVABILITY_SECRET_ENV_PATH="$PROJECT_OVERLAY_DIR/secrets/observability.env" if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then GRAFANA_ADMIN_PASSWORD="$(python3 - <<'PY' import secrets print(secrets.token_urlsafe(24)) PY )" echo "GRAFANA_ADMIN_PASSWORD not provided; generated a random bootstrap password for Grafana admin user '$GRAFANA_ADMIN_USERNAME'" >&2 fi cat > "$PROJECT_SECRET_ENV_PATH" < "$PROJECT_OVERLAY_DIR/secrets/forgejo.env" < "$OBSERVABILITY_SECRET_ENV_PATH" <&2 elif [[ -n "${HEADLAMP_ADMIN_TOKEN_PASS:-}" ]]; then store_secret_to_pass "$HEADLAMP_ADMIN_TOKEN_PASS" "$HEADLAMP_ADMIN_TOKEN" echo "stored headlamp admin token in pass: $HEADLAMP_ADMIN_TOKEN_PASS" fi forgejo_admin_user_b64=$(printf '%s' "$FORGEJO_ADMIN_USERNAME" | base64 | tr -d '\n') forgejo_admin_pass_b64=$(printf '%s' "$FORGEJO_ADMIN_PASSWORD" | base64 | tr -d '\n') forgejo_admin_email_b64=$(printf '%s' "$FORGEJO_ADMIN_EMAIL" | base64 | tr -d '\n') forgejo_runner_name_b64=$(printf '%s' "$FORGEJO_RUNNER_NAME" | base64 | tr -d '\n') forgejo_runner_labels_b64=$(printf '%s' "$FORGEJO_RUNNER_LABELS" | base64 | tr -d '\n') kubectl -n forgejo exec -i deploy/forgejo -- /bin/bash --noprofile --norc -s <&2 exit 1 fi if su-exec git /usr/local/bin/forgejo admin user list | awk 'NR>1 {print \$2}' | grep -qx "\$ADMIN_USERNAME"; then echo "forgejo admin already exists: \$ADMIN_USERNAME" else su-exec git /usr/local/bin/forgejo admin user create --config "\$APP_INI" --admin --username "\$ADMIN_USERNAME" --password "\$ADMIN_PASSWORD" --email "\$ADMIN_EMAIL" --must-change-password=false fi EOF RUNNER_NAME="$(printf '%s' "$forgejo_runner_name_b64" | base64 -d)" RUNNER_LABELS="$(printf '%s' "$forgejo_runner_labels_b64" | base64 -d)" RUNNER_TOKEN="$(kubectl -n forgejo exec deploy/forgejo -- /bin/bash --noprofile --norc -lc 'su-exec git /usr/local/bin/forgejo --config /data/gitea/conf/app.ini actions generate-runner-token' | tr -d '\r\n')" kubectl -n forgejo delete job forgejo-runner-bootstrap --ignore-not-found cat <- mkdir -p /data && if [ ! -s /data/.runner ]; then forgejo-runner register --no-interactive --name "$RUNNER_NAME" --instance "$FORGEJO_INTERNAL_URL" --token "$RUNNER_TOKEN" --labels "$RUNNER_LABELS"; else echo runner config already exists; fi volumeMounts: - name: forgejo-data mountPath: /data subPath: forgejo-runner EOF kubectl -n forgejo wait --for=condition=complete --timeout=300s job/forgejo-runner-bootstrap kubectl -n forgejo rollout restart deployment/forgejo-runner kubectl -n forgejo rollout status deployment/forgejo-runner --timeout=300s FORGEJO_BOOTSTRAP_PORT="${FORGEJO_BOOTSTRAP_PORT:-$(python3 - <<'PY' import socket s=socket.socket() s.bind(('127.0.0.1', 0)) print(s.getsockname()[1]) s.close() PY )}" FORGEJO_BOOTSTRAP_URL="http://127.0.0.1:${FORGEJO_BOOTSTRAP_PORT}" kubectl -n forgejo port-forward svc/forgejo "${FORGEJO_BOOTSTRAP_PORT}:3000" >/tmp/forgejo-port-forward.log 2>&1 & FORGEJO_PORT_FORWARD_PID=$! trap 'if [[ -n "${FORGEJO_PORT_FORWARD_PID:-}" ]]; then kill "$FORGEJO_PORT_FORWARD_PID" >/dev/null 2>&1 || true; fi; if [[ -n "${K3S_TUNNEL_PID:-}" ]]; then kill "$K3S_TUNNEL_PID" >/dev/null 2>&1 || true; fi' EXIT wait_for_url "$FORGEJO_BOOTSTRAP_URL" "Forgejo bootstrap URL" 60 2 if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then FORGEJO_ADMIN_API_TOKEN="$(kubectl -n forgejo exec deploy/forgejo -- /bin/bash --noprofile --norc -lc "su-exec git /usr/local/bin/forgejo admin user generate-access-token --config /data/gitea/conf/app.ini --username '$FORGEJO_ADMIN_USERNAME' --token-name bootstrap-$(date +%s) --scopes read:user,read:repository,write:repository,write:user --raw" | tr -d '\r\n')" forgejo_bootstrap_args=( --forgejo-url "$FORGEJO_BOOTSTRAP_URL" --token "$FORGEJO_ADMIN_API_TOKEN" --admin-username "$FORGEJO_ADMIN_USERNAME" --repo-owner "$FORGEJO_REPO_OWNER" --repo-name "$FORGEJO_REPO_NAME" --kubeconfig "$KUBECONFIG_PATH" --registry-username "$REGISTRY_USERNAME" --registry-password "$REGISTRY_PASSWORD" --registry-host "$REGISTRY_DOMAIN" --project-name "$PROJECT_NAME" --project-namespace "$PROJECT_NAMESPACE" --project-deployments "${PROJECT_DEPLOYMENTS// /,}" --project-path "$PROJECT_REPO_PATH" ) if [[ "$FORGEJO_REPO_PRIVATE" == "true" ]]; then forgejo_bootstrap_args+=(--repo-private) fi python3 "$ROOT_DIR/scripts/hetzner/forgejo-bootstrap.py" "${forgejo_bootstrap_args[@]}" FORGEJO_PUSH_URL_BASE="$FORGEJO_BOOTSTRAP_URL" bash "$ROOT_DIR/scripts/hetzner/seed-forgejo-repo.sh" wait_for_url "$FORGEJO_ROOT_URL" "Forgejo public URL" 180 5 wait_for_http_status "https://$REGISTRY_DOMAIN/v2/" "registry public URL" '200|401' 180 5 else docker build -t "$BOOTSTRAP_IMAGE" "$PROJECT_DIR" docker save "$BOOTSTRAP_IMAGE" \ | ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$SSH_TARGET" 'sudo k3s ctr images import -' for deployment in $PROJECT_DEPLOYMENTS; do kubectl -n "$PROJECT_NAMESPACE" set image "deployment/${deployment}" app="$BOOTSTRAP_IMAGE" done for deployment in $PROJECT_DEPLOYMENTS; do kubectl -n "$PROJECT_NAMESPACE" rollout status "deployment/${deployment}" --timeout=180s done fi DURABLE_K3S_API_URL="$K3S_API_URL" DURABLE_INSECURE_SKIP_TLS_VERIFY=0 if [[ "$USE_SSH_TUNNEL_FOR_K3S" == "1" ]]; then if [[ -n "${TAILSCALE_CONTROL_PLANE_HOSTNAME:-}" ]]; then DURABLE_K3S_API_URL="https://${TAILSCALE_CONTROL_PLANE_HOSTNAME}:6443" elif [[ "$TF_ADMIN_CIDR_BLOCKS" != '[]' ]]; then DURABLE_K3S_API_URL="https://${SERVER_IP}:6443" DURABLE_INSECURE_SKIP_TLS_VERIFY=1 fi fi python3 - "$KUBECONFIG_PATH" "$DURABLE_K3S_API_URL" "$DURABLE_INSECURE_SKIP_TLS_VERIFY" <<'PY' import sys import yaml path, server, insecure = sys.argv[1], sys.argv[2], sys.argv[3] == '1' config = yaml.safe_load(open(path)) cluster = config['clusters'][0]['cluster'] cluster['server'] = server if insecure: cluster.pop('certificate-authority-data', None) cluster['insecure-skip-tls-verify'] = True else: cluster.pop('insecure-skip-tls-verify', None) yaml.safe_dump(config, open(path, 'w'), sort_keys=False) PY K3S_API_URL="$DURABLE_K3S_API_URL" echo "bootstrap complete" echo "project_name=$PROJECT_NAME" echo "project_namespace=$PROJECT_NAMESPACE" echo "project_overlay_dir=$PROJECT_OVERLAY_DIR" echo "server_ip=$SERVER_IP" echo "ssh_target=$SSH_TARGET" echo "k3s_api_url=$K3S_API_URL" echo "kubeconfig=$KUBECONFIG_PATH" echo "ci_kubeconfig=$CI_KUBECONFIG_PATH" echo "bootstrap_delivery_mode=$BOOTSTRAP_DELIVERY_MODE" echo "forgejo_url=$FORGEJO_ROOT_URL" echo "forgejo_repo=${FORGEJO_ROOT_URL%/}/$FORGEJO_REPO_OWNER/$FORGEJO_REPO_NAME" echo "registry_url=https://$REGISTRY_DOMAIN" echo "grafana_url=$GRAFANA_ROOT_URL" echo "headlamp_url=https://$HEADLAMP_DOMAIN/" echo "headlamp_token_pass=${HEADLAMP_ADMIN_TOKEN_PASS:-}" echo "dns_provider=${CLOUDFLARE_API_TOKEN:+cloudflare}${PORKBUN_API_KEY:+porkbun}"