feat: automate forgejo bootstrap with pass-backed secrets

.env.example

@@ -13,28 +13,27 @@ EXECUTOR_STATE_DIR=/var/lib/unrip/executor-state
 
 # Repo-driven Hetzner bootstrap values live separately from the app .env.
 # Copy scripts/hetzner/bootstrap-secrets.env.example to
-# scripts/hetzner/bootstrap-secrets.env, fill in the values, then:
+# scripts/hetzner/bootstrap-secrets.env, configure non-secret values plus *_PASS
+# mappings to your pass store, then:
 #   source scripts/hetzner/bootstrap-secrets.env
 #   bash scripts/hetzner/bootstrap.sh
 #
-# The local-machine bootstrap flow is:
+# Canonical operator flow uses `pass` for sensitive values; explicit env vars still
-# 1. provide Hetzner token + SSH key path + DNS/ingress values + app/bootstrap secrets
+# override pass-backed lookups for CI/testing.
-# 2. run Terraform from infra/terraform/hetzner
-# 3. wait for cloud-init/k3s readiness
-# 4. fetch kubeconfig to .state/hetzner/kubeconfig.yaml
-# 5. create Kubernetes Secrets from local values
-# 6. build/import the current app image into k3s
-# 7. apply repo Kubernetes manifests and let the bootstrap job create topics
 #
-# Expected bootstrap inputs:
+# Expected bootstrap inputs now include:
-# - HCLOUD_TOKEN
+# - HCLOUD_TOKEN_PASS or HCLOUD_TOKEN
 # - SSH_PUBLIC_KEY_PATH
-# - TF_ADMIN_CIDR_BLOCKS
+# - PUBLIC_DOMAIN
 # - BASE_DOMAIN
-# - FORGEJO_DOMAIN
+# - LETSENCRYPT_EMAIL
-# - FORGEJO_ROOT_URL
+# - REGISTRY_USERNAME
-# - NEAR_INTENTS_API_KEY
+# - REGISTRY_PASSWORD_PASS or REGISTRY_PASSWORD
-# - FORGEJO_RUNNER_REGISTRATION_TOKEN
+# - NEAR_INTENTS_API_KEY_PASS or NEAR_INTENTS_API_KEY
+# - FORGEJO_ADMIN_USERNAME
+# - FORGEJO_ADMIN_EMAIL
+# - FORGEJO_ADMIN_PASSWORD_PASS or FORGEJO_ADMIN_PASSWORD
+# - optional DNS provider creds via *_PASS or direct env vars
 #
 # Future k3s deployment should source the app values from Kubernetes Secret/ConfigMap.
 # Hetzner bootstrap path clones the repo to /opt/unrip/repo for later deploy/k8s assets.

.forgejo/workflows/deploy.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+  workflow_dispatch:
 
 jobs:
   deploy:
@@ -14,6 +15,7 @@ jobs:
       PROJECT_NAME: ${{ vars.PROJECT_NAME || 'unrip' }}
       PROJECT_NAMESPACE: ${{ vars.PROJECT_NAMESPACE || vars.PROJECT_NAME || 'unrip' }}
       PROJECT_DEPLOYMENTS: ${{ vars.PROJECT_DEPLOYMENTS || 'near-intents-ingest,dummy-reactor,dummy-executor,dummy-consumer' }}
+      PROJECT_REGISTRY_SECRET_NAME: ${{ vars.PROJECT_REGISTRY_SECRET_NAME || format('{0}-registry-creds', vars.PROJECT_NAME || 'unrip') }}
       REPO_CLONE_URL: ${{ github.server_url }}/${{ github.repository }}.git
     steps:
       - name: Install tooling
@@ -35,6 +37,7 @@ jobs:
             echo "BUILD_JOB=$BUILD_JOB"
             echo "PROJECT_NAMESPACE=$PROJECT_NAMESPACE"
             echo "PROJECT_DEPLOYMENTS=$PROJECT_DEPLOYMENTS"
+            echo "PROJECT_REGISTRY_SECRET_NAME=$PROJECT_REGISTRY_SECRET_NAME"
           } >> "$GITHUB_ENV"
 
       - name: Build and push image in-cluster
@@ -58,7 +61,7 @@ jobs:
                     emptyDir: {}
                   - name: registry-creds
                     secret:
-                      secretName: unrip-registry-creds
+                      secretName: ${PROJECT_REGISTRY_SECRET_NAME}
                       items:
                         - key: .dockerconfigjson
                           path: config.json

docs/hetzner-k3s-bootstrap.md

@@ -38,39 +38,48 @@ Goal: provision and deploy everything from this repo to a single Hetzner machine
 - `docker`
 - `curl`
 - `python3`
+- `git`
+- `pass`
 
 ## Required local env
 Start from:
 
 ```bash
 cp scripts/hetzner/bootstrap-secrets.env.example scripts/hetzner/bootstrap-secrets.env
+${EDITOR:-vi} scripts/hetzner/bootstrap-secrets.env
 source scripts/hetzner/bootstrap-secrets.env
 ```
 
+The mapping file should contain non-secret config plus `pass` entry references for secrets. Bootstrap and destroy load the first line from each configured pass entry without echoing it. Explicit env exports still override `pass` lookups.
+
 Required values:
-- `HCLOUD_TOKEN`
+- `HCLOUD_TOKEN_PASS` or `HCLOUD_TOKEN`
 - `SSH_PUBLIC_KEY_PATH`
 - `PUBLIC_DOMAIN`
 - `BASE_DOMAIN`
 - recommended Tailscale values:
-  - `TAILSCALE_AUTH_KEY`
+  - `TAILSCALE_AUTH_KEY_PASS` or `TAILSCALE_AUTH_KEY`
   - `TAILSCALE_CONTROL_PLANE_HOSTNAME`
 - `FORGEJO_DOMAIN`
 - `FORGEJO_ROOT_URL`
 - `REGISTRY_DOMAIN`
 - `LETSENCRYPT_EMAIL`
 - `REGISTRY_USERNAME`
-- `REGISTRY_PASSWORD`
+- `REGISTRY_PASSWORD_PASS` or `REGISTRY_PASSWORD`
-- `NEAR_INTENTS_API_KEY`
+- `NEAR_INTENTS_API_KEY_PASS` or `NEAR_INTENTS_API_KEY`
-- `FORGEJO_RUNNER_REGISTRATION_TOKEN`
+- `FORGEJO_ADMIN_USERNAME`
+- `FORGEJO_ADMIN_EMAIL`
+- `FORGEJO_ADMIN_PASSWORD_PASS` or `FORGEJO_ADMIN_PASSWORD`
+- optional generated-secret target: `FORGEJO_RUNNER_REGISTRATION_TOKEN_PASS`
+- optional repo settings: `FORGEJO_REPO_OWNER`, `FORGEJO_REPO_NAME`, `FORGEJO_REPO_PRIVATE`
 
 Optional for automatic DNS:
 - Cloudflare:
-  - `CLOUDFLARE_API_TOKEN`
+  - `CLOUDFLARE_API_TOKEN_PASS` or `CLOUDFLARE_API_TOKEN`
-  - `CLOUDFLARE_ZONE_ID`
+  - `CLOUDFLARE_ZONE_ID_PASS` or `CLOUDFLARE_ZONE_ID`
 - Porkbun:
-  - `PORKBUN_API_KEY`
+  - `PORKBUN_API_KEY_PASS` or `PORKBUN_API_KEY`
-  - `PORKBUN_SECRET_API_KEY`
+  - `PORKBUN_SECRET_API_KEY_PASS` or `PORKBUN_SECRET_API_KEY`
 
 ## Bootstrap
 ```bash
@@ -82,9 +91,15 @@ Outputs:
 - Tailscale joined if configured
 - k3s installed
 - kubeconfig written to `.state/hetzner/kubeconfig.yaml`
-- overlay secrets and ingress host patches rendered from local env
+- CI kubeconfig written to `.state/hetzner/kubeconfig.incluster.yaml`
+- overlay secrets and ingress host patches rendered from local env / `pass`
 - namespaces, Redpanda, app deployments, Forgejo, registry, ingress, cert-manager, and issuers applied
-- bootstrap image built and first rollout triggered
+- Forgejo admin account created automatically if missing
+- Forgejo runner registration token generated automatically and stored in the live Kubernetes secret
+- Forgejo repository created automatically
+- Forgejo Actions secrets and variables configured automatically
+- repo pushed to Forgejo automatically in the default `forgejo-actions` delivery mode
+- first deployment triggered from Forgejo Actions by default
 
 ## Tailscale-first admin access
 Recommended mode:
@@ -113,29 +128,40 @@ bash scripts/k8s/logs.sh
 ```
 
 ## Self-hosted CI/CD handoff
-After bootstrap:
+Default bootstrap now automates the Forgejo handoff:
-1. open Forgejo at `https://${FORGEJO_DOMAIN}`
+1. create the Forgejo repo
-2. seed or mirror this repo into Forgejo
+2. configure the repository Actions secrets:
-3. add Forgejo Actions secrets:
    - `KUBECONFIG_B64`
    - `REGISTRY_USERNAME`
    - `REGISTRY_PASSWORD`
-4. add Forgejo Actions variable:
+3. configure the repository Actions variables:
    - `REGISTRY_HOST=${REGISTRY_DOMAIN}`
-5. push to `main`
+   - `PROJECT_NAME`
+   - `PROJECT_NAMESPACE`
+   - `PROJECT_DEPLOYMENTS`
+4. push the current repo to `main`
 
 The workflow then:
-- builds the image
+- starts a Kubernetes Job in the target namespace
-- pushes it to `https://${REGISTRY_DOMAIN}`
+- uses Kaniko plus the Kubernetes registry auth secret to build and push `${REGISTRY_DOMAIN}/${PROJECT_NAME}:${GIT_SHA}`
-- updates the app deployments in `unrip`
+- updates the app deployments in `PROJECT_NAMESPACE`
 - waits for rollout
 
+Legacy local-image bootstrap remains available with:
+
+```bash
+BOOTSTRAP_DELIVERY_MODE=local-image-import bash scripts/hetzner/bootstrap.sh
+```
+
 ## Destroy everything
 ```bash
+source scripts/hetzner/bootstrap-secrets.env
 bash scripts/hetzner/destroy.sh
 ```
 
+`destroy.sh` reads `HCLOUD_TOKEN` and optional `TAILSCALE_AUTH_KEY` via the same `*_PASS` mapping mechanism as bootstrap.
+
 ## Current limitations
-- Forgejo admin bootstrap and repo seeding are still operator-driven after the first cluster bootstrap.
+- automated repo creation currently assumes `FORGEJO_REPO_OWNER == FORGEJO_ADMIN_USERNAME`
-- bootstrap and CI authentication paths should still be hardened before production use.
+- bootstrap still uses local `docker` to generate the registry htpasswd secret
-- routine deploys are intended to be registry-native through Forgejo Actions, but that still needs a real-world verification pass.
+- bootstrap and CI authentication paths should still be hardened before production use

docs/hetzner-self-hosted-ci-runbook.md

@@ -8,15 +8,24 @@ From your workstation:
 ```bash
 cp scripts/hetzner/bootstrap-secrets.env.example scripts/hetzner/bootstrap-secrets.env
 source scripts/hetzner/bootstrap-secrets.env
+python3 -c 'import nacl'  # verify PyNaCl is installed for Actions secret encryption
 bash scripts/hetzner/bootstrap.sh
 ```
 
+`scripts/hetzner/bootstrap-secrets.env` should contain non-secret bootstrap settings and `pass` entry mappings like `HCLOUD_TOKEN_PASS`, `REGISTRY_PASSWORD_PASS`, and `FORGEJO_ADMIN_PASSWORD_PASS`. If you explicitly export the raw env vars, they override the `pass` lookups.
+
 After that you should have:
 - `.state/hetzner/kubeconfig.yaml`
+- `.state/hetzner/kubeconfig.incluster.yaml`
 - Forgejo reachable at `https://${FORGEJO_DOMAIN}`
+- the target Forgejo repo created automatically
+- repository Actions secrets/variables populated for CI
+- the current repo pushed to Forgejo automatically in default mode
 - Registry reachable at `https://${REGISTRY_DOMAIN}`
 - private admin/control-plane access over Tailscale if configured
 
+Bootstrap repo automation requires `FORGEJO_ADMIN_USERNAME`, `FORGEJO_ADMIN_PASSWORD`, and Python `PyNaCl` locally so the script can encrypt Forgejo Actions secrets before upload. The same bootstrap flow now also creates the initial Forgejo admin account and generates the one-time runner registration token after Forgejo is up.
+
 ## Verify the cluster
 ```bash
 export KUBECONFIG=$PWD/.state/hetzner/kubeconfig.yaml
@@ -28,52 +37,59 @@ kubectl -n unrip get deploy,pods
 ```
 
 ## Seed the repo into Forgejo
-Create the target repo in Forgejo, then from your workstation:
+Default bootstrap already seeds the repo with:
 
 ```bash
-git remote add forgejo https://${FORGEJO_DOMAIN}/<owner>/<repo>.git
+bash scripts/hetzner/seed-forgejo-repo.sh
-git push forgejo main
 ```
 
+You only need to run it manually if you skipped seeding during bootstrap or want to push again after local changes.
+
 ## Configure Forgejo Actions secrets and variables
-Create these repository secrets in Forgejo:
+Bootstrap upserts these repository secrets automatically:
 - `KUBECONFIG_B64`
 - `REGISTRY_USERNAME`
 - `REGISTRY_PASSWORD`
 
-Create these repository variables:
+Bootstrap upserts these repository variables automatically:
 - `REGISTRY_HOST=${REGISTRY_DOMAIN}`
-- optional: `PROJECT_NAME=unrip`
+- `PROJECT_NAME=${PROJECT_NAME}`
-- optional: `PROJECT_NAMESPACE=unrip`
+- `PROJECT_NAMESPACE=${PROJECT_NAMESPACE}`
-- optional: `PROJECT_DEPLOYMENTS=near-intents-ingest,dummy-reactor,dummy-executor,dummy-consumer`
+- `PROJECT_DEPLOYMENTS` as a comma-separated version of the bootstrap deployment list
 
-Generate `KUBECONFIG_B64` from the bootstrap kubeconfig:
+The Forgejo repo configuration step is idempotent, so rerunning bootstrap updates the same repo secrets/variables in place.
-
-```bash
-base64 -w0 .state/hetzner/kubeconfig.yaml
-```
 
 ## Workflow behavior
 The workflow in `.forgejo/workflows/deploy.yml` now:
-1. installs `buildah` and `kubectl` on the Forgejo runner
+1. installs `kubectl` on the Forgejo runner
-2. checks out the repo with the Forgejo job token
+2. loads kubeconfig from `KUBECONFIG_B64`
-3. loads kubeconfig from `KUBECONFIG_B64`
+3. computes `IMAGE=${REGISTRY_HOST}/${PROJECT_NAME}:${GIT_SHA}`
-4. logs into the private registry
+4. creates an in-cluster Kubernetes Job in `PROJECT_NAMESPACE`
-5. builds `registry.<domain>/<project-name>:${GIT_SHA}` with `buildah`
+5. that Job checks out the repo with the Forgejo job token in an init container
-6. pushes the image
+6. Kaniko builds and pushes the image using the Kubernetes registry auth secret
-7. updates each deployment listed in `PROJECT_DEPLOYMENTS` inside `PROJECT_NAMESPACE`
+7. the workflow updates each deployment listed in `PROJECT_DEPLOYMENTS` inside `PROJECT_NAMESPACE`
-8. waits for rollout after each image update
+8. the workflow waits for rollout after each image update
 
 Default behavior if you do not set project variables:
 - `PROJECT_NAME=unrip`
 - `PROJECT_NAMESPACE=unrip`
 - `PROJECT_DEPLOYMENTS=near-intents-ingest,dummy-reactor,dummy-executor,dummy-consumer`
+- `PROJECT_REGISTRY_SECRET_NAME=unrip-registry-creds`
 
 For a future project, reuse the same workflow by changing only the Forgejo repository variables instead of copying the workflow.
 
-The first bootstrap deploy is different from routine CI:
+Default bootstrap now uses the same routine CI path for the first deploy:
-- bootstrap fetches the real kubeconfig from the node and imports a local bootstrap image directly into k3s
+- bootstrap fetches the real kubeconfig from the node
-- routine CI is intended to push versioned images to the private registry
+- bootstrap derives an in-cluster kubeconfig for the runner
+- bootstrap creates the Forgejo repo and Actions config
+- bootstrap pushes to `main`
+- Forgejo Actions builds the image in-cluster and deploys it
+
+Legacy mode still exists if you explicitly set:
+
+```bash
+BOOTSTRAP_DELIVERY_MODE=local-image-import
+```
 
 ## Trigger deploys
 Push to `main` in Forgejo:
@@ -103,6 +119,7 @@ Currently supported DNS providers:
 TLS is issued by cert-manager using the rendered Let's Encrypt email and ingress hosts.
 
 ## Current limitations
-- Forgejo admin bootstrap and repository creation are not yet API-automated.
+- the bootstrap path now creates the initial admin account and one-time runner registration token automatically from inside the Forgejo pod, but it still depends on the operator supplying the intended admin credentials up front
-- Forgejo repository secrets/variables still need to be populated before the first real deploy run.
+- runner registration no longer needs a pre-seeded Kubernetes secret, but the runner config still lives on `emptyDir`, so bootstrap must recreate `/data/.runner` after a runner pod replacement
-- The runner currently uses host-mode jobs and installs `buildah`/`kubectl` at job start, which is functional but not yet optimized.
+- automated repo creation currently assumes `FORGEJO_REPO_OWNER == FORGEJO_ADMIN_USERNAME`
+- the runner currently uses host-mode jobs and installs `kubectl` at job start; the image build itself runs in-cluster via Kaniko, which is functional but not yet optimized

scripts/hetzner/bootstrap-secrets.env.example

@@ -1,48 +1,75 @@
-# Copy this file to scripts/hetzner/bootstrap-secrets.env and fill in the values.
+# Copy to scripts/hetzner/bootstrap-secrets.env, adjust the non-secret values and
-# Then run: source scripts/hetzner/bootstrap-secrets.env
+# pass entry paths, then source it before running bootstrap or destroy:
+#   source scripts/hetzner/bootstrap-secrets.env
+#   bash scripts/hetzner/bootstrap.sh
+#
+# Canonical operator path:
+# - set *_PASS variables to pass entry paths
+# - optionally override a value directly via ENV for CI/tests or one-off debugging
+# - bootstrap/destroy will load the first line from each pass entry without echoing it
 
-export HCLOUD_TOKEN=replace_me
+export PASS_PREFIX="infra/unrip3"
-export SSH_PUBLIC_KEY_PATH="$HOME/.ssh/id_ed25519.pub"
+pass_ref() {
+  printf '%s/%s' "$PASS_PREFIX" "$1"
+}
+
+# Required infra access
+export HCLOUD_TOKEN_PASS="${HCLOUD_TOKEN_PASS:-$(pass_ref hetzner/hcloud-token)}"
+export SSH_PUBLIC_KEY_PATH="${SSH_PUBLIC_KEY_PATH:-$HOME/.ssh/id_ed25519.pub}"
 
 # Optional project override. Defaults target the built-in unrip project overlay.
-export PROJECT_NAME=unrip
+export PROJECT_NAME="${PROJECT_NAME:-unrip}"
-export PROJECT_NAMESPACE=unrip
+export PROJECT_NAMESPACE="${PROJECT_NAMESPACE:-$PROJECT_NAME}"
 # export PROJECT_OVERLAY_DIR="$PWD/deploy/k8s/overlays/hetzner-single-node"
 # export PROJECT_KUSTOMIZE_PATH="../../projects/unrip/base"
-# export PROJECT_SECRET_NAME=unrip-secrets
+# export PROJECT_SECRET_NAME="unrip-secrets"
-# export PROJECT_SECRET_ENV_BASENAME=unrip.env
+# export PROJECT_SECRET_ENV_BASENAME="unrip.env"
-# export PROJECT_REGISTRY_SECRET_NAME=unrip-registry-creds
+# export PROJECT_REGISTRY_SECRET_NAME="unrip-registry-creds"
-# export PROJECT_IMAGE_REPOSITORY=unrip
+# export PROJECT_IMAGE_REPOSITORY="unrip"
 # export PROJECT_DEPLOYMENTS="near-intents-ingest dummy-reactor dummy-executor dummy-consumer"
 
 # Tailscale-first admin access (recommended)
-export TAILSCALE_AUTH_KEY=
+export TAILSCALE_AUTH_KEY_PASS="${TAILSCALE_AUTH_KEY_PASS:-$(pass_ref tailscale/auth-key)}"
-# optional override; leave empty to auto-discover the node via local `tailscale status --json`
+# Optional override; leave empty to auto-discover the node via local `tailscale status --json`.
-export TAILSCALE_CONTROL_PLANE_HOSTNAME=
+export TAILSCALE_CONTROL_PLANE_HOSTNAME="${TAILSCALE_CONTROL_PLANE_HOSTNAME:-}"
 
-# Optional fallback if you want public admin ports instead of Tailscale
+# Optional fallback if you intentionally want public SSH/Kubernetes admin exposure.
-export TF_ADMIN_CIDR_BLOCKS='[]'
+export TF_ADMIN_CIDR_BLOCKS="${TF_ADMIN_CIDR_BLOCKS:-[]}"
 
 # Public naming for ingress/TLS
-export PUBLIC_DOMAIN=unrip-bootstrap.example.com
+export PUBLIC_DOMAIN="${PUBLIC_DOMAIN:-doran.133011.xyz}"
-export BASE_DOMAIN=example.com
+export BASE_DOMAIN="${BASE_DOMAIN:-133011.xyz}"
-export FORGEJO_DOMAIN=git.example.com
+export FORGEJO_DOMAIN="${FORGEJO_DOMAIN:-git.${BASE_DOMAIN}}"
-export FORGEJO_ROOT_URL=https://git.example.com/
+export FORGEJO_ROOT_URL="${FORGEJO_ROOT_URL:-https://${FORGEJO_DOMAIN}/}"
-export REGISTRY_DOMAIN=registry.example.com
+export REGISTRY_DOMAIN="${REGISTRY_DOMAIN:-registry.${BASE_DOMAIN}}"
-export LETSENCRYPT_EMAIL=ops@example.com
+export LETSENCRYPT_EMAIL="${LETSENCRYPT_EMAIL:-ops@example.com}"
 
 # Optional DNS automation: choose one provider
 # Cloudflare
-export CLOUDFLARE_API_TOKEN=
+export CLOUDFLARE_API_TOKEN_PASS="${CLOUDFLARE_API_TOKEN_PASS:-$(pass_ref cloudflare/api-token)}"
-export CLOUDFLARE_ZONE_ID=
+export CLOUDFLARE_ZONE_ID_PASS="${CLOUDFLARE_ZONE_ID_PASS:-$(pass_ref cloudflare/zone-id)}"
 # Porkbun
-export PORKBUN_API_KEY=
+export PORKBUN_API_KEY_PASS="${PORKBUN_API_KEY_PASS:-$(pass_ref porkbun/api-key)}"
-export PORKBUN_SECRET_API_KEY=
+export PORKBUN_SECRET_API_KEY_PASS="${PORKBUN_SECRET_API_KEY_PASS:-$(pass_ref porkbun/secret-api-key)}"
 
 # Registry auth for CI/CD and image pulls
-export REGISTRY_USERNAME=unrip
+export REGISTRY_USERNAME="${REGISTRY_USERNAME:-unrip}"
-export REGISTRY_PASSWORD=replace_me
+export REGISTRY_PASSWORD_PASS="${REGISTRY_PASSWORD_PASS:-$(pass_ref registry/password)}"
 
-# Application and bootstrap secrets
+# Application secret
-export NEAR_INTENTS_API_KEY=replace_me
+export NEAR_INTENTS_API_KEY_PASS="${NEAR_INTENTS_API_KEY_PASS:-$(pass_ref near-intents/api-key)}"
-export FORGEJO_RUNNER_REGISTRATION_TOKEN=replace_me
+
+# Forgejo bootstrap
+export FORGEJO_ADMIN_USERNAME="${FORGEJO_ADMIN_USERNAME:-forgejo-admin}"
+export FORGEJO_ADMIN_EMAIL="${FORGEJO_ADMIN_EMAIL:-${FORGEJO_ADMIN_USERNAME}@${BASE_DOMAIN}}"
+export FORGEJO_ADMIN_PASSWORD_PASS="${FORGEJO_ADMIN_PASSWORD_PASS:-$(pass_ref forgejo/admin-password)}"
+
+# Optional explicit overrides for CI/testing:
+# export HCLOUD_TOKEN="..."
+# export REGISTRY_PASSWORD="..."
+# export NEAR_INTENTS_API_KEY="..."
+# export FORGEJO_ADMIN_PASSWORD="..."
+# export CLOUDFLARE_API_TOKEN="..."
+# export CLOUDFLARE_ZONE_ID="..."
+# export PORKBUN_API_KEY="..."
+# export PORKBUN_SECRET_API_KEY="..."

scripts/hetzner/bootstrap.sh

@@ -2,134 +2,45 @@
 set -euo pipefail
 
 ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd)
+# shellcheck disable=SC1091
+source "$ROOT_DIR/scripts/hetzner/lib.sh"
+load_bootstrap_env
+
 TF_DIR="$ROOT_DIR/infra/terraform/hetzner"
 STATE_DIR="$ROOT_DIR/.state/hetzner"
 KUBECONFIG_PATH="$STATE_DIR/kubeconfig.yaml"
+CI_KUBECONFIG_PATH="$STATE_DIR/kubeconfig.incluster.yaml"
 OVERLAY_DIR="$ROOT_DIR/deploy/k8s/overlays/hetzner-single-node"
 DEFAULT_PROJECT_NAME="unrip"
 DEFAULT_PROJECT_NAMESPACE="$DEFAULT_PROJECT_NAME"
 mkdir -p "$STATE_DIR"
 
-require() {
-  command -v "$1" >/dev/null 2>&1 || { echo "missing command: $1" >&2; exit 1; }
-}
-
-wait_for_url() {
-  local url="$1"
-  local label="$2"
-  local max_attempts="${3:-120}"
-  local sleep_seconds="${4:-5}"
-  local attempt=1
-
-  until curl -kfsS "$url" >/dev/null 2>&1; do
-    if (( attempt >= max_attempts )); then
-      echo "timed out waiting for ${label}: ${url}" >&2
-      return 1
-    fi
-    if (( attempt == 1 || attempt % 6 == 0 )); then
-      echo "waiting for ${label} (${attempt}/${max_attempts})..."
-    fi
-    sleep "$sleep_seconds"
-    attempt=$((attempt + 1))
-  done
-}
-
-wait_for_ssh() {
-  local target="$1"
-  local max_attempts="${2:-120}"
-  local sleep_seconds="${3:-5}"
-  local attempt=1
-
-  until ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 "$target" 'echo ssh-ready' >/dev/null 2>&1; do
-    if (( attempt >= max_attempts )); then
-      echo "timed out waiting for ssh: ${target}" >&2
-      return 1
-    fi
-    if (( attempt == 1 || attempt % 6 == 0 )); then
-      echo "waiting for ssh (${attempt}/${max_attempts})..."
-    fi
-    sleep "$sleep_seconds"
-    attempt=$((attempt + 1))
-  done
-}
-
-wait_for_tailscale_node() {
-  local host_name="$1"
-  local max_attempts="${2:-120}"
-  local sleep_seconds="${3:-5}"
-  local attempt=1
-
-  command -v tailscale >/dev/null 2>&1 || {
-    echo "tailscale CLI is required locally for tailscale-first bootstrap" >&2
-    return 1
-  }
-
-  while true; do
-    local discovered
-    discovered=$(tailscale status --json 2>/dev/null | python3 - "$host_name" <<'PY'
-import json,sys
-host=sys.argv[1]
-try:
-    data=json.load(sys.stdin)
-except Exception:
-    print("")
-    raise SystemExit(0)
-peers=data.get('Peer',{})
-matches=[]
-for peer in peers.values():
-    if peer.get('HostName') == host:
-        matches.append(peer)
-for peer in sorted(matches, key=lambda p: ((p.get('Online') is True), p.get('DNSName') or ''), reverse=True):
-    if peer.get('Online'):
-        dns=(peer.get('DNSName') or '').rstrip('.')
-        if dns:
-            print(dns)
-            raise SystemExit(0)
-for peer in sorted(matches, key=lambda p: p.get('DNSName') or '', reverse=True):
-    if peer.get('TailscaleIPs'):
-        print(peer['TailscaleIPs'][0])
-        raise SystemExit(0)
-print("")
-PY
-)
-    if [[ -n "$discovered" ]]; then
-      printf '%s\n' "$discovered"
-      return 0
-    fi
-    if (( attempt >= max_attempts )); then
-      echo "timed out waiting for tailscale node: ${host_name}" >&2
-      return 1
-    fi
-    if (( attempt == 1 || attempt % 6 == 0 )); then
-      echo "waiting for tailscale node ${host_name} (${attempt}/${max_attempts})..." >&2
-    fi
-    sleep "$sleep_seconds"
-    attempt=$((attempt + 1))
-  done
-}
-
 require terraform
 require kubectl
-require docker
 require curl
 require python3
 require ssh
-require realpath
+require git
+require docker
+require base64
+
+resolve_secret_var HCLOUD_TOKEN required
+resolve_secret_var TAILSCALE_AUTH_KEY optional
+resolve_secret_var NEAR_INTENTS_API_KEY required
+resolve_secret_var REGISTRY_PASSWORD required
+resolve_secret_var FORGEJO_ADMIN_PASSWORD required
+resolve_secret_var CLOUDFLARE_API_TOKEN optional
+resolve_secret_var PORKBUN_API_KEY optional
+resolve_secret_var PORKBUN_SECRET_API_KEY optional
 
-: "${HCLOUD_TOKEN:?set HCLOUD_TOKEN}"
 : "${SSH_PUBLIC_KEY_PATH:?set SSH_PUBLIC_KEY_PATH}"
 : "${PUBLIC_DOMAIN:?set PUBLIC_DOMAIN}"
 : "${LETSENCRYPT_EMAIL:?set LETSENCRYPT_EMAIL}"
-: "${TAILSCALE_AUTH_KEY:=}"
-: "${TAILSCALE_CONTROL_PLANE_HOSTNAME:=}"
-: "${NEAR_INTENTS_API_KEY:?set NEAR_INTENTS_API_KEY}"
 : "${BASE_DOMAIN:?set BASE_DOMAIN}"
 : "${FORGEJO_DOMAIN:=git.${BASE_DOMAIN}}"
 : "${FORGEJO_ROOT_URL:=https://${FORGEJO_DOMAIN}/}"
 : "${REGISTRY_DOMAIN:=registry.${BASE_DOMAIN}}"
 : "${REGISTRY_USERNAME:?set REGISTRY_USERNAME}"
-: "${REGISTRY_PASSWORD:?set REGISTRY_PASSWORD}"
-: "${FORGEJO_RUNNER_REGISTRATION_TOKEN:?set FORGEJO_RUNNER_REGISTRATION_TOKEN}"
 : "${TF_ADMIN_CIDR_BLOCKS:=}"
 : "${PROJECT_NAME:=$DEFAULT_PROJECT_NAME}"
 : "${PROJECT_NAMESPACE:=$DEFAULT_PROJECT_NAMESPACE}"
@@ -142,31 +53,38 @@ require realpath
 : "${PROJECT_REGISTRY_SECRET_NAME:=${PROJECT_NAME}-registry-creds}"
 : "${PROJECT_IMAGE_REPOSITORY:=${PROJECT_NAME}}"
 : "${PROJECT_DEPLOYMENTS:=near-intents-ingest dummy-reactor dummy-executor dummy-consumer}"
+: "${FORGEJO_ADMIN_USERNAME:?set FORGEJO_ADMIN_USERNAME}"
+: "${FORGEJO_ADMIN_EMAIL:?set FORGEJO_ADMIN_EMAIL}"
+: "${FORGEJO_RUNNER_NAME:=k3s-runner}"
+: "${FORGEJO_RUNNER_LABELS:=linux-amd64:host}"
+: "${FORGEJO_REPO_OWNER:=$FORGEJO_ADMIN_USERNAME}"
+: "${FORGEJO_REPO_NAME:=$(basename "$ROOT_DIR")}"
+: "${FORGEJO_REPO_PRIVATE:=true}"
+: "${BOOTSTRAP_DELIVERY_MODE:=forgejo-actions}"
 
 BOOTSTRAP_IMAGE="${PROJECT_IMAGE_REPOSITORY}:bootstrap"
 PROJECT_SECRET_ENV_PATH="$PROJECT_OVERLAY_DIR/secrets/$PROJECT_SECRET_ENV_BASENAME"
 GENERATED_OVERLAY_DIR="$STATE_DIR/generated-overlay"
-GENERATED_OVERLAY_KUSTOMIZATION="$GENERATED_OVERLAY_DIR/kustomization.yaml"
-
 SSH_PUBLIC_KEY=$(cat "$SSH_PUBLIC_KEY_PATH")
 SSH_PRIVATE_KEY_PATH="${SSH_PUBLIC_KEY_PATH%.pub}"
 if [[ ! -f "$SSH_PRIVATE_KEY_PATH" ]]; then
   echo "missing ssh private key for bootstrap: $SSH_PRIVATE_KEY_PATH" >&2
   exit 1
 fi
+
 TF_VARS=(
   -var "hcloud_token=$HCLOUD_TOKEN"
   -var "ssh_public_key=$SSH_PUBLIC_KEY"
   -var "public_domain=$PUBLIC_DOMAIN"
   -var "bootstrap_repo_url=local-bootstrap"
-  -var "tailscale_auth_key=$TAILSCALE_AUTH_KEY"
+  -var "tailscale_auth_key=${TAILSCALE_AUTH_KEY:-}"
-  -var "tailscale_control_plane_hostname=$TAILSCALE_CONTROL_PLANE_HOSTNAME"
+  -var "tailscale_control_plane_hostname=${TAILSCALE_CONTROL_PLANE_HOSTNAME:-}"
 )
 if [[ -n "$TF_ADMIN_CIDR_BLOCKS" && "$TF_ADMIN_CIDR_BLOCKS" != '[]' ]]; then
   TF_VARS+=(-var "admin_cidr_blocks=$TF_ADMIN_CIDR_BLOCKS")
 fi
 
-if [[ -n "$TAILSCALE_AUTH_KEY" ]]; then
+if [[ -n "${TAILSCALE_AUTH_KEY:-}" ]]; then
   bash "$ROOT_DIR/scripts/hetzner/print-tailscale-firewall-note.sh"
 fi
 
@@ -177,7 +95,7 @@ fi
 
 SERVER_IP=$(terraform -chdir="$TF_DIR" output -raw server_ipv4)
 K3S_API_URL=$(terraform -chdir="$TF_DIR" output -raw k3s_api_url)
-if [[ -n "$TAILSCALE_AUTH_KEY" ]]; then
+if [[ -n "${TAILSCALE_AUTH_KEY:-}" ]]; then
   DISCOVERED_TAILSCALE_HOST="${TAILSCALE_CONTROL_PLANE_HOSTNAME:-$(wait_for_tailscale_node "$BOOTSTRAP_NODE_NAME")}" 
   SSH_TARGET="root@${DISCOVERED_TAILSCALE_HOST}"
   K3S_API_URL="https://${DISCOVERED_TAILSCALE_HOST}:6443"
@@ -203,6 +121,15 @@ ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile
   | sed "s|https://127.0.0.1:6443|${K3S_API_URL}|" > "$KUBECONFIG_PATH"
 export KUBECONFIG="$KUBECONFIG_PATH"
 
+python3 - "$KUBECONFIG_PATH" "$CI_KUBECONFIG_PATH" <<'PY'
+import sys
+import yaml
+src, dst = sys.argv[1], sys.argv[2]
+config = yaml.safe_load(open(src))
+config['clusters'][0]['cluster']['server'] = 'https://kubernetes.default.svc:443'
+yaml.safe_dump(config, open(dst, 'w'), sort_keys=False)
+PY
+
 mkdir -p "$PROJECT_OVERLAY_DIR/secrets" "$GENERATED_OVERLAY_DIR"
 cat > "$PROJECT_SECRET_ENV_PATH" <<EOF
 NEAR_INTENTS_API_KEY=$NEAR_INTENTS_API_KEY
@@ -210,7 +137,6 @@ EOF
 cat > "$PROJECT_OVERLAY_DIR/secrets/forgejo.env" <<EOF
 root_url=$FORGEJO_ROOT_URL
 domain=$FORGEJO_DOMAIN
-runner_registration_token=$FORGEJO_RUNNER_REGISTRATION_TOKEN
 EOF
 python3 - <<PY
 from pathlib import Path
@@ -277,16 +203,82 @@ kubectl -n "$PROJECT_NAMESPACE" create secret docker-registry "$PROJECT_REGISTRY
   --dry-run=client -o yaml | kubectl apply -f -
 kubectl apply -k "$GENERATED_OVERLAY_DIR"
 
-docker build -t "$BOOTSTRAP_IMAGE" "$ROOT_DIR"
+kubectl -n forgejo rollout status deployment/forgejo --timeout=300s
-docker save "$BOOTSTRAP_IMAGE" \
+kubectl -n registry rollout status deployment/registry --timeout=300s
+kubectl -n "$PROJECT_NAMESPACE" rollout status deployment/redpanda --timeout=300s
+kubectl -n forgejo wait --for=condition=available deployment/forgejo-runner --timeout=300s || true
+
+forgejo_admin_user_b64=$(printf '%s' "$FORGEJO_ADMIN_USERNAME" | base64 | tr -d '\n')
+forgejo_admin_pass_b64=$(printf '%s' "$FORGEJO_ADMIN_PASSWORD" | base64 | tr -d '\n')
+forgejo_admin_email_b64=$(printf '%s' "$FORGEJO_ADMIN_EMAIL" | base64 | tr -d '\n')
+forgejo_runner_name_b64=$(printf '%s' "$FORGEJO_RUNNER_NAME" | base64 | tr -d '\n')
+forgejo_runner_labels_b64=$(printf '%s' "$FORGEJO_RUNNER_LABELS" | base64 | tr -d '\n')
+kubectl -n forgejo exec -i deploy/forgejo -- /bin/bash --noprofile --norc -s <<EOF
+set -euo pipefail
+ADMIN_USERNAME=\$(printf '%s' '$forgejo_admin_user_b64' | base64 -d)
+ADMIN_PASSWORD=\$(printf '%s' '$forgejo_admin_pass_b64' | base64 -d)
+ADMIN_EMAIL=\$(printf '%s' '$forgejo_admin_email_b64' | base64 -d)
+RUNNER_NAME=\$(printf '%s' '$forgejo_runner_name_b64' | base64 -d)
+RUNNER_LABELS=\$(printf '%s' '$forgejo_runner_labels_b64' | base64 -d)
+APP_INI=/data/gitea/conf/app.ini
+if [[ ! -f "$APP_INI" ]]; then
+  echo "missing Forgejo config at $APP_INI" >&2
+  exit 1
+fi
+if su-exec git /usr/local/bin/forgejo admin user list | awk 'NR>1 {print \$2}' | grep -qx "\$ADMIN_USERNAME"; then
+  echo "forgejo admin already exists: \$ADMIN_USERNAME"
+else
+  su-exec git /usr/local/bin/forgejo admin user create --config "$APP_INI" --admin --username "\$ADMIN_USERNAME" --password "\$ADMIN_PASSWORD" --email "\$ADMIN_EMAIL" --must-change-password=false
+fi
+if [[ ! -f /data/.runner ]]; then
+  RUNNER_TOKEN=\$(su-exec git /usr/local/bin/forgejo --config "$APP_INI" actions generate-runner-token)
+  forgejo-runner register --no-interactive --name "\$RUNNER_NAME" --instance "$FORGEJO_ROOT_URL" --token "\$RUNNER_TOKEN" --labels "\$RUNNER_LABELS"
+  install -o 1000 -g 1000 -m 600 .runner /data/.runner
+  rm -f .runner
+  echo "registered forgejo runner config at /data/.runner"
+else
+  echo "forgejo runner already configured: /data/.runner"
+fi
+EOF
+kubectl -n forgejo rollout restart deployment/forgejo-runner
+kubectl -n forgejo rollout status deployment/forgejo-runner --timeout=300s
+
+if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then
+  wait_for_url "$FORGEJO_ROOT_URL" "Forgejo public URL" 180 5
+  wait_for_http_status "https://$REGISTRY_DOMAIN/v2/" "registry public URL" '200|401' 180 5
+
+  forgejo_bootstrap_args=(
+    --forgejo-url "$FORGEJO_ROOT_URL"
+    --admin-username "$FORGEJO_ADMIN_USERNAME"
+    --admin-password "$FORGEJO_ADMIN_PASSWORD"
+    --repo-owner "$FORGEJO_REPO_OWNER"
+    --repo-name "$FORGEJO_REPO_NAME"
+    --kubeconfig "$KUBECONFIG_PATH"
+    --registry-username "$REGISTRY_USERNAME"
+    --registry-password "$REGISTRY_PASSWORD"
+    --registry-host "$REGISTRY_DOMAIN"
+    --project-name "$PROJECT_NAME"
+    --project-namespace "$PROJECT_NAMESPACE"
+    --project-deployments "${PROJECT_DEPLOYMENTS// /,}"
+  )
+  if [[ "$FORGEJO_REPO_PRIVATE" == "true" ]]; then
+    forgejo_bootstrap_args+=(--repo-private)
+  fi
+  python3 "$ROOT_DIR/scripts/hetzner/forgejo-bootstrap.py" "${forgejo_bootstrap_args[@]}"
+
+  bash "$ROOT_DIR/scripts/hetzner/seed-forgejo-repo.sh"
+else
+  docker build -t "$BOOTSTRAP_IMAGE" "$ROOT_DIR"
+  docker save "$BOOTSTRAP_IMAGE" \
     | ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$SSH_TARGET" 'sudo k3s ctr images import -'
 
-for deployment in $PROJECT_DEPLOYMENTS; do
+  for deployment in $PROJECT_DEPLOYMENTS; do
     kubectl -n "$PROJECT_NAMESPACE" set image "deployment/${deployment}" app="$BOOTSTRAP_IMAGE"
-done
+  done
-for deployment in $PROJECT_DEPLOYMENTS; do
+  for deployment in $PROJECT_DEPLOYMENTS; do
     kubectl -n "$PROJECT_NAMESPACE" rollout status "deployment/${deployment}" --timeout=180s
-done
+  done
+fi
 
 echo "bootstrap complete"
 echo "project_name=$PROJECT_NAME"
@@ -296,7 +288,9 @@ echo "server_ip=$SERVER_IP"
 echo "ssh_target=$SSH_TARGET"
 echo "k3s_api_url=$K3S_API_URL"
 echo "kubeconfig=$KUBECONFIG_PATH"
-echo "bootstrap_image=$BOOTSTRAP_IMAGE"
+echo "ci_kubeconfig=$CI_KUBECONFIG_PATH"
+echo "bootstrap_delivery_mode=$BOOTSTRAP_DELIVERY_MODE"
 echo "forgejo_url=$FORGEJO_ROOT_URL"
+echo "forgejo_repo=${FORGEJO_ROOT_URL%/}/$FORGEJO_REPO_OWNER/$FORGEJO_REPO_NAME"
 echo "registry_url=https://$REGISTRY_DOMAIN"
 echo "dns_provider=${CLOUDFLARE_API_TOKEN:+cloudflare}${PORKBUN_API_KEY:+porkbun}"

scripts/hetzner/destroy.sh

@@ -2,12 +2,19 @@
 set -euo pipefail
 
 ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd)
+# shellcheck disable=SC1091
+source "$ROOT_DIR/scripts/hetzner/lib.sh"
+load_bootstrap_env
+
 TF_DIR="$ROOT_DIR/infra/terraform/hetzner"
 
-: "${HCLOUD_TOKEN:?set HCLOUD_TOKEN}"
+require terraform
+
+resolve_secret_var HCLOUD_TOKEN required
+resolve_secret_var TAILSCALE_AUTH_KEY optional
+
 : "${SSH_PUBLIC_KEY_PATH:?set SSH_PUBLIC_KEY_PATH}"
 : "${PUBLIC_DOMAIN:=bootstrap.example.com}"
-: "${TAILSCALE_AUTH_KEY:=}"
 : "${TAILSCALE_CONTROL_PLANE_HOSTNAME:=}"
 : "${TF_ADMIN_CIDR_BLOCKS:=}"
 
@@ -16,7 +23,7 @@ TF_VARS=(
   -var "hcloud_token=$HCLOUD_TOKEN"
   -var "ssh_public_key=$SSH_PUBLIC_KEY"
   -var "public_domain=$PUBLIC_DOMAIN"
-  -var "tailscale_auth_key=$TAILSCALE_AUTH_KEY"
+  -var "tailscale_auth_key=${TAILSCALE_AUTH_KEY:-}"
   -var "tailscale_control_plane_hostname=$TAILSCALE_CONTROL_PLANE_HOSTNAME"
 )
 

scripts/hetzner/forgejo-bootstrap.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+import argparse
+import base64
+import json
+import ssl
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+import yaml
+from nacl import encoding, public
+
+
+class ForgejoClient:
+    def __init__(self, base_url: str, username: str, password: str):
+        self.base_url = base_url.rstrip('/')
+        credentials = base64.b64encode(f'{username}:{password}'.encode()).decode()
+        self.headers = {
+            'Authorization': f'Basic {credentials}',
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+        }
+        self.ssl_context = ssl.create_default_context()
+
+    def request(self, method: str, path: str, payload=None, expected=(200, 201, 204)):
+        url = f'{self.base_url}{path}'
+        data = None
+        if payload is not None:
+            data = json.dumps(payload).encode()
+        req = urllib.request.Request(url, data=data, method=method)
+        for key, value in self.headers.items():
+            req.add_header(key, value)
+        try:
+            with urllib.request.urlopen(req, context=self.ssl_context) as response:
+                body = response.read().decode() if response.length != 0 else ''
+                if response.status not in expected:
+                    raise RuntimeError(f'{method} {path} returned {response.status}: {body}')
+                if not body:
+                    return None
+                return json.loads(body)
+        except urllib.error.HTTPError as exc:
+            body = exc.read().decode()
+            if exc.code not in expected:
+                raise RuntimeError(f'{method} {path} returned {exc.code}: {body}') from exc
+            return json.loads(body) if body else None
+
+    def get_repo(self, owner: str, repo: str):
+        try:
+            return self.request('GET', f'/api/v1/repos/{owner}/{repo}')
+        except RuntimeError as exc:
+            if ' returned 404:' in str(exc):
+                return None
+            raise
+
+    def create_repo(self, name: str, private: bool):
+        return self.request('POST', '/api/v1/user/repos', {
+            'name': name,
+            'private': private,
+            'auto_init': False,
+            'default_branch': 'main',
+        }, expected=(201,))
+
+    def upsert_variable(self, owner: str, repo: str, name: str, value: str):
+        try:
+            self.request('POST', f'/api/v1/repos/{owner}/{repo}/actions/variables/{urllib.parse.quote(name)}', {
+                'value': value,
+            }, expected=(201,))
+        except RuntimeError as exc:
+            if ' returned 409:' not in str(exc) and ' returned 422:' not in str(exc):
+                raise
+            self.request('PUT', f'/api/v1/repos/{owner}/{repo}/actions/variables/{urllib.parse.quote(name)}', {
+                'value': value,
+            }, expected=(201, 204))
+
+    def upsert_secret(self, owner: str, repo: str, name: str, value: str):
+        key = self.request('GET', f'/api/v1/repos/{owner}/{repo}/actions/secrets/public-key')
+        public_key = public.PublicKey(key['key'].encode(), encoder=encoding.Base64Encoder)
+        sealed_box = public.SealedBox(public_key)
+        encrypted_value = base64.b64encode(sealed_box.encrypt(value.encode())).decode()
+        self.request('PUT', f'/api/v1/repos/{owner}/{repo}/actions/secrets/{urllib.parse.quote(name)}', {
+            'encrypted_value': encrypted_value,
+            'key_id': key['key_id'],
+        }, expected=(201, 204))
+
+
+def render_ci_kubeconfig(source: Path) -> str:
+    config = yaml.safe_load(source.read_text())
+    clusters = config.get('clusters') or []
+    if not clusters:
+        raise SystemExit('kubeconfig does not contain any clusters')
+    clusters[0]['cluster']['server'] = 'https://kubernetes.default.svc:443'
+    return yaml.safe_dump(config, sort_keys=False)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Bootstrap Forgejo repo secrets/variables for CI/CD')
+    parser.add_argument('--forgejo-url', required=True)
+    parser.add_argument('--admin-username', required=True)
+    parser.add_argument('--admin-password', required=True)
+    parser.add_argument('--repo-owner', required=True)
+    parser.add_argument('--repo-name', required=True)
+    parser.add_argument('--repo-private', action='store_true', default=False)
+    parser.add_argument('--kubeconfig', required=True)
+    parser.add_argument('--registry-username', required=True)
+    parser.add_argument('--registry-password', required=True)
+    parser.add_argument('--registry-host', required=True)
+    parser.add_argument('--project-name', required=True)
+    parser.add_argument('--project-namespace', required=True)
+    parser.add_argument('--project-deployments', required=True)
+    args = parser.parse_args()
+
+    client = ForgejoClient(args.forgejo_url, args.admin_username, args.admin_password)
+    repo = client.get_repo(args.repo_owner, args.repo_name)
+    if repo is None:
+        created = client.create_repo(args.repo_name, args.repo_private)
+        print(f'created repo {created["full_name"]}')
+    else:
+        print(f'repo already exists: {repo["full_name"]}')
+
+    ci_kubeconfig = render_ci_kubeconfig(Path(args.kubeconfig))
+    client.upsert_secret(args.repo_owner, args.repo_name, 'KUBECONFIG_B64', base64.b64encode(ci_kubeconfig.encode()).decode())
+    client.upsert_secret(args.repo_owner, args.repo_name, 'REGISTRY_USERNAME', args.registry_username)
+    client.upsert_secret(args.repo_owner, args.repo_name, 'REGISTRY_PASSWORD', args.registry_password)
+    print('upserted repo action secrets')
+
+    client.upsert_variable(args.repo_owner, args.repo_name, 'REGISTRY_HOST', args.registry_host)
+    client.upsert_variable(args.repo_owner, args.repo_name, 'PROJECT_NAME', args.project_name)
+    client.upsert_variable(args.repo_owner, args.repo_name, 'PROJECT_NAMESPACE', args.project_namespace)
+    client.upsert_variable(args.repo_owner, args.repo_name, 'PROJECT_DEPLOYMENTS', args.project_deployments)
+    print('upserted repo action variables')
+
+
+if __name__ == '__main__':
+    main()

scripts/hetzner/lib.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+BOOTSTRAP_ENV_FILE_DEFAULT="$ROOT_DIR/scripts/hetzner/bootstrap-secrets.env"
+
+require() {
+  command -v "$1" >/dev/null 2>&1 || { echo "missing command: $1" >&2; exit 1; }
+}
+
+load_bootstrap_env() {
+  local env_file="${BOOTSTRAP_ENV_FILE:-$BOOTSTRAP_ENV_FILE_DEFAULT}"
+  if [[ -f "$env_file" ]]; then
+    # shellcheck disable=SC1090
+    source "$env_file"
+  fi
+}
+
+pass_get_first_line() {
+  local path="$1"
+  pass show "$path" | awk 'NR==1 { print; exit }'
+}
+
+resolve_secret_var() {
+  local name="$1"
+  local mode="${2:-required}"
+  local ref_name="${name}_PASS"
+  local value="${!name:-}"
+  local ref="${!ref_name:-}"
+
+  if [[ -z "$value" && -n "$ref" ]]; then
+    require pass
+    value="$(pass_get_first_line "$ref")"
+    printf -v "$name" '%s' "$value"
+    export "$name"
+  fi
+
+  if [[ "$mode" == "required" && -z "${!name:-}" ]]; then
+    echo "set $name or $ref_name" >&2
+    exit 1
+  fi
+}
+
+store_secret_to_pass() {
+  local path="$1"
+  local value="$2"
+
+  [[ -n "$path" ]] || return 0
+  require pass
+  printf '%s\n' "$value" | pass insert -m -f "$path" >/dev/null
+}
+
+wait_for_url() {
+  local url="$1"
+  local label="$2"
+  local max_attempts="${3:-120}"
+  local sleep_seconds="${4:-5}"
+  local attempt=1
+
+  until curl -kfsS "$url" >/dev/null 2>&1; do
+    if (( attempt >= max_attempts )); then
+      echo "timed out waiting for ${label}: ${url}" >&2
+      return 1
+    fi
+    if (( attempt == 1 || attempt % 6 == 0 )); then
+      echo "waiting for ${label} (${attempt}/${max_attempts})..."
+    fi
+    sleep "$sleep_seconds"
+    attempt=$((attempt + 1))
+  done
+}
+
+wait_for_http_status() {
+  local url="$1"
+  local label="$2"
+  local expected_regex="$3"
+  local max_attempts="${4:-120}"
+  local sleep_seconds="${5:-5}"
+  local attempt=1
+
+  while true; do
+    local status
+    status="$(curl -ksS -o /dev/null -w '%{http_code}' "$url" || true)"
+    if [[ "$status" =~ ^(${expected_regex})$ ]]; then
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "timed out waiting for ${label}: ${url} (last status=${status:-n/a})" >&2
+      return 1
+    fi
+    if (( attempt == 1 || attempt % 6 == 0 )); then
+      echo "waiting for ${label} (${attempt}/${max_attempts}) status=${status:-n/a}..."
+    fi
+    sleep "$sleep_seconds"
+    attempt=$((attempt + 1))
+  done
+}
+
+wait_for_ssh() {
+  local target="$1"
+  local max_attempts="${2:-120}"
+  local sleep_seconds="${3:-5}"
+  local attempt=1
+
+  until ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 "$target" 'echo ssh-ready' >/dev/null 2>&1; do
+    if (( attempt >= max_attempts )); then
+      echo "timed out waiting for ssh: ${target}" >&2
+      return 1
+    fi
+    if (( attempt == 1 || attempt % 6 == 0 )); then
+      echo "waiting for ssh (${attempt}/${max_attempts})..."
+    fi
+    sleep "$sleep_seconds"
+    attempt=$((attempt + 1))
+  done
+}
+
+wait_for_tailscale_node() {
+  local host_name="$1"
+  local max_attempts="${2:-120}"
+  local sleep_seconds="${3:-5}"
+  local attempt=1
+
+  command -v tailscale >/dev/null 2>&1 || {
+    echo "tailscale CLI is required locally for tailscale-first bootstrap" >&2
+    return 1
+  }
+
+  while true; do
+    local discovered
+    discovered=$(tailscale status --json 2>/dev/null | python3 - "$host_name" <<'PY'
+import json,sys
+host=sys.argv[1]
+try:
+    data=json.load(sys.stdin)
+except Exception:
+    print("")
+    raise SystemExit(0)
+peers=data.get('Peer',{})
+matches=[]
+for peer in peers.values():
+    if peer.get('HostName') == host:
+        matches.append(peer)
+for peer in sorted(matches, key=lambda p: ((p.get('Online') is True), p.get('DNSName') or ''), reverse=True):
+    if peer.get('Online'):
+        dns=(peer.get('DNSName') or '').rstrip('.')
+        if dns:
+            print(dns)
+            raise SystemExit(0)
+for peer in sorted(matches, key=lambda p: p.get('DNSName') or '', reverse=True):
+    if peer.get('TailscaleIPs'):
+        print(peer['TailscaleIPs'][0])
+        raise SystemExit(0)
+print("")
+PY
+)
+    if [[ -n "$discovered" ]]; then
+      printf '%s\n' "$discovered"
+      return 0
+    fi
+    if (( attempt >= max_attempts )); then
+      echo "timed out waiting for tailscale node: ${host_name}" >&2
+      return 1
+    fi
+    if (( attempt == 1 || attempt % 6 == 0 )); then
+      echo "waiting for tailscale node ${host_name} (${attempt}/${max_attempts})..." >&2
+    fi
+    sleep "$sleep_seconds"
+    attempt=$((attempt + 1))
+  done
+}

scripts/hetzner/seed-forgejo-repo.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd)
+# shellcheck disable=SC1091
+source "$ROOT_DIR/scripts/hetzner/lib.sh"
+load_bootstrap_env
+
+resolve_secret_var FORGEJO_ADMIN_PASSWORD required
+
+: "${FORGEJO_ROOT_URL:?set FORGEJO_ROOT_URL}"
+: "${FORGEJO_ADMIN_USERNAME:?set FORGEJO_ADMIN_USERNAME}"
+: "${FORGEJO_REPO_OWNER:=$FORGEJO_ADMIN_USERNAME}"
+: "${FORGEJO_REPO_NAME:=$(basename "$ROOT_DIR")}"
+: "${FORGEJO_PUSH_REMOTE_NAME:=forgejo}"
+: "${FORGEJO_PUSH_REF:=HEAD:refs/heads/main}"
+
+require git
+
+remote_url="${FORGEJO_ROOT_URL%/}/${FORGEJO_REPO_OWNER}/${FORGEJO_REPO_NAME}.git"
+current_remote_url="$(git remote get-url "$FORGEJO_PUSH_REMOTE_NAME" 2>/dev/null || true)"
+if [[ -z "$current_remote_url" ]]; then
+  git remote add "$FORGEJO_PUSH_REMOTE_NAME" "$remote_url"
+elif [[ "$current_remote_url" != "$remote_url" ]]; then
+  git remote set-url "$FORGEJO_PUSH_REMOTE_NAME" "$remote_url"
+fi
+
+askpass_script="$(mktemp)"
+trap 'rm -f "$askpass_script"' EXIT
+cat > "$askpass_script" <<'EOF'
+#!/usr/bin/env sh
+case "$1" in
+  *sername*) printf '%s\n' "$FORGEJO_ADMIN_USERNAME" ;;
+  *assword*) printf '%s\n' "$FORGEJO_ADMIN_PASSWORD" ;;
+  *) exit 1 ;;
+esac
+EOF
+chmod 700 "$askpass_script"
+
+GIT_TERMINAL_PROMPT=0 \
+GIT_ASKPASS="$askpass_script" \
+FORGEJO_ADMIN_USERNAME="$FORGEJO_ADMIN_USERNAME" \
+FORGEJO_ADMIN_PASSWORD="$FORGEJO_ADMIN_PASSWORD" \
+  git push "$FORGEJO_PUSH_REMOTE_NAME" "$FORGEJO_PUSH_REF"
+
+echo "seeded ${remote_url}"