fix: harden hetzner rebuild bootstrap flow

.forgejo/workflows/deploy.yml

@@ -78,9 +78,9 @@ jobs:
                     command: ["/bin/sh", "-lc"]
                     args:
                       - >-
-                        git clone --depth=1 "https://oauth2:${REPO_TOKEN}@${REPO_CLONE_URL#https://}" /workspace &&
+                        git -c credential.username=oauth2 -c http.extraHeader="Authorization: Bearer ${REPO_TOKEN}" clone --depth=1 "${REPO_CLONE_URL}" /workspace &&
                         cd /workspace &&
-                        git fetch --depth=1 origin "${GITHUB_SHA}" &&
+                        git -c credential.username=oauth2 -c http.extraHeader="Authorization: Bearer ${REPO_TOKEN}" fetch --depth=1 origin "${GITHUB_SHA}" &&
                         git checkout --detach "${GITHUB_SHA}"
                     volumeMounts:
                       - name: workspace

deploy/k8s/platform/base/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
+  - traefik-config.yaml
   - forgejo.yaml
   - forgejo-rbac.yaml
   - forgejo-runner.yaml

deploy/k8s/platform/base/traefik-config.yaml

@@ -0,0 +1,14 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: traefik
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    service:
+      type: ClusterIP
+    ports:
+      web:
+        hostPort: 80
+      websecure:
+        hostPort: 443

docs/hetzner-k3s-bootstrap.md

@@ -5,21 +5,22 @@ Goal: provision and deploy everything from this repo to a single Hetzner machine
 ## Stack
 - Terraform provisions the Hetzner Cloud VM, private network, and firewall
 - cloud-init installs Tailscale first when configured, then installs k3s automatically
+- cloud-init leaves only a bootstrap marker on the node; it does not clone this repo or apply Kubernetes assets
 - Kubernetes manifests deploy:
   - Redpanda
   - trading system services
   - private registry
   - Forgejo
-  - ingress-nginx
+  - k3s-bundled Traefik ingress resources
   - cert-manager
   - ACME issuers
 - local bootstrap script:
   - runs Terraform
   - optionally creates DNS records via Cloudflare or Porkbun
-  - writes overlay secrets/host patches from local env
-  - applies the Hetzner single-node k8s overlay
-  - builds the current app image locally
   - fetches the real kubeconfig from the node
+  - writes overlay secrets/host patches from local env
+  - applies the Hetzner single-node k8s overlay from the operator workstation checkout
+  - builds the current app image locally
   - imports the bootstrap image into k3s for the first rollout
 
 ## Files
@@ -33,13 +34,24 @@ Goal: provision and deploy everything from this repo to a single Hetzner machine
 - `.forgejo/workflows/deploy.yml`
 
 ## Required local tools
+Always required:
 - `terraform`
 - `kubectl`
-- `docker`
 - `curl`
 - `python3`
+- `ssh`
 - `git`
-- `pass`
+- `base64`
+- `realpath`
+- `pass` when using any `*_PASS` mapping
+
+Conditionally required:
+- `docker` only for `BOOTSTRAP_DELIVERY_MODE=local-image-import`, or as a fallback when no native `htpasswd` binary is available locally
+- `htpasswd` is preferred for registry secret generation and avoids the docker fallback
+
+Required local Python modules:
+- `PyYAML` (`python3 -m pip install PyYAML`) for kubeconfig rendering during bootstrap
+- `PyNaCl` (`python3 -m pip install PyNaCl`) only when `BOOTSTRAP_DELIVERY_MODE=forgejo-actions` so bootstrap can encrypt Forgejo Actions secrets
 
 ## Required local env
 Start from:
@@ -52,6 +64,15 @@ source scripts/hetzner/bootstrap-secrets.env
 
 The mapping file should contain non-secret config plus `pass` entry references for secrets. Bootstrap and destroy load the first line from each configured pass entry without echoing it. Explicit env exports still override `pass` lookups.
 
+When you run `scripts/hetzner/bootstrap.sh`, it uses this file to materialize local Kubernetes inputs before apply:
+- overwrites `deploy/k8s/overlays/hetzner-single-node/secrets/unrip.env` with `NEAR_INTENTS_API_KEY`
+- overwrites `deploy/k8s/overlays/hetzner-single-node/secrets/forgejo.env` with Forgejo `root_url` and `domain`
+- renders generated ingress and issuer patch files under `.state/hetzner/generated-overlay/`
+- creates `registry-secrets` in namespace `registry` from `REGISTRY_USERNAME` and `REGISTRY_PASSWORD`
+- creates the project docker-registry pull secret in `PROJECT_NAMESPACE` from the same registry credentials
+
+This is different from running `kubectl apply -k deploy/k8s/overlays/hetzner-single-node` manually: plain Kustomize apply only consumes the checked-in overlay files and only generates `unrip-secrets` and `forgejo-secrets`. It does not create registry auth secrets and does not read `scripts/hetzner/bootstrap-secrets.env` on its own.
+
 Required values:
 - `HCLOUD_TOKEN_PASS` or `HCLOUD_TOKEN`
 - `SSH_PUBLIC_KEY_PATH`
@@ -59,7 +80,8 @@ Required values:
 - `BASE_DOMAIN`
 - recommended Tailscale values:
   - `TAILSCALE_AUTH_KEY_PASS` or `TAILSCALE_AUTH_KEY`
-  - `TAILSCALE_CONTROL_PLANE_HOSTNAME`
+  - optional `TAILSCALE_CONTROL_PLANE_HOSTNAME` to force a stable Tailscale DNS name for kube access
+  - if `TAILSCALE_CONTROL_PLANE_HOSTNAME` is left empty, bootstrap auto-discovers the node via local `tailscale status --json`
 - `FORGEJO_DOMAIN`
 - `FORGEJO_ROOT_URL`
 - `REGISTRY_DOMAIN`
@@ -70,7 +92,6 @@ Required values:
 - `FORGEJO_ADMIN_USERNAME`
 - `FORGEJO_ADMIN_EMAIL`
 - `FORGEJO_ADMIN_PASSWORD_PASS` or `FORGEJO_ADMIN_PASSWORD`
-- optional generated-secret target: `FORGEJO_RUNNER_REGISTRATION_TOKEN_PASS`
 - optional repo settings: `FORGEJO_REPO_OWNER`, `FORGEJO_REPO_NAME`, `FORGEJO_REPO_PRIVATE`
 
 Optional for automatic DNS:
@@ -90,15 +111,16 @@ Outputs:
 - Hetzner VM created
 - Tailscale joined if configured
 - k3s installed
+- cloud-init writes `/opt/unrip/bootstrap/README.txt` as a marker that node-local repo bootstrap is not active yet
 - kubeconfig written to `.state/hetzner/kubeconfig.yaml`
 - CI kubeconfig written to `.state/hetzner/kubeconfig.incluster.yaml`
 - overlay secrets and ingress host patches rendered from local env / `pass`
-- namespaces, Redpanda, app deployments, Forgejo, registry, ingress, cert-manager, and issuers applied
+- namespaces, Redpanda, app deployments, Forgejo, registry, Traefik-targeted ingress resources, cert-manager, and issuers applied
 - Forgejo admin account created automatically if missing
-- Forgejo runner registration token generated automatically and stored in the live Kubernetes secret
+- Forgejo runner registration is generated automatically from inside the Forgejo pod and the resulting `/data/.runner` config is stored under the shared `forgejo-data` persistent volume used by the runner deployment
-- Forgejo repository created automatically
+- Forgejo repository created automatically in either the admin user's namespace or a pre-existing organization named by `FORGEJO_REPO_OWNER`
 - Forgejo Actions secrets and variables configured automatically
-- repo pushed to Forgejo automatically in the default `forgejo-actions` delivery mode
+- repo pushed to Forgejo automatically in the default `forgejo-actions` delivery mode via authenticated HTTPS Git push
 - first deployment triggered from Forgejo Actions by default
 
 ## Tailscale-first admin access
@@ -120,6 +142,8 @@ Supported scripted providers:
 - Porkbun
 
 TLS is handled in-cluster by cert-manager using Let's Encrypt issuers and the rendered ingress hosts.
+The platform base assumes the default k3s Traefik ingress controller is present; it does not install ingress-nginx.
+For clean-cluster applies, the base kustomization now includes cert-manager before the `ClusterIssuer` resources so the issuer CRs can be created in the same bootstrap flow.
 
 ## Observe the cluster
 ```bash
@@ -129,7 +153,7 @@ bash scripts/k8s/logs.sh
 
 ## Self-hosted CI/CD handoff
 Default bootstrap now automates the Forgejo handoff:
-1. create the Forgejo repo
+1. create the Forgejo repo in the admin namespace or in a pre-existing organization named by `FORGEJO_REPO_OWNER`
 2. configure the repository Actions secrets:
    - `KUBECONFIG_B64`
    - `REGISTRY_USERNAME`
@@ -143,6 +167,7 @@ Default bootstrap now automates the Forgejo handoff:
 
 The workflow then:
 - starts a Kubernetes Job in the target namespace
+- checks out the repo inside that Job using the Forgejo job token via `Authorization: Bearer ...` HTTP auth
 - uses Kaniko plus the Kubernetes registry auth secret to build and push `${REGISTRY_DOMAIN}/${PROJECT_NAME}:${GIT_SHA}`
 - updates the app deployments in `PROJECT_NAMESPACE`
 - waits for rollout
@@ -154,14 +179,46 @@ BOOTSTRAP_DELIVERY_MODE=local-image-import bash scripts/hetzner/bootstrap.sh
 ```
 
 ## Destroy everything
+Default destroy only removes Terraform-managed Hetzner infrastructure:
+
 ```bash
 source scripts/hetzner/bootstrap-secrets.env
 bash scripts/hetzner/destroy.sh
 ```
 
-`destroy.sh` reads `HCLOUD_TOKEN` and optional `TAILSCALE_AUTH_KEY` via the same `*_PASS` mapping mechanism as bootstrap.
+Opt-in flags make destructive cleanup of bootstrap-managed leftovers explicit:
+
+```bash
+source scripts/hetzner/bootstrap-secrets.env
+DESTROY_DNS=true \
+DESTROY_LOCAL_STATE=true \
+DESTROY_FORGEJO_REPO=true \
+bash scripts/hetzner/destroy.sh
+```
+
+`destroy.sh` reads `HCLOUD_TOKEN`, optional `TAILSCALE_AUTH_KEY`, optional DNS provider credentials, and optional Forgejo admin credentials via the same `*_PASS` mapping mechanism as bootstrap.
+It uses the same Terraform inputs as bootstrap for the infrastructure resources, then can optionally:
+- delete the scripted DNS records for `${BASE_DOMAIN}`, `git.${BASE_DOMAIN}`, and `registry.${BASE_DOMAIN}`
+- remove local bootstrap artifacts under `.state/hetzner/`, `deploy/k8s/overlays/hetzner-single-node/generated/`, and the local Terraform working/state files in `infra/terraform/hetzner/`
+- delete the bootstrap-managed Forgejo repository via the Forgejo API
+
+Supported scripted DNS cleanup providers:
+- Cloudflare
+- Porkbun
+
+Cleanup defaults are intentionally conservative:
+- `DESTROY_DNS=false` keeps provider records unless you explicitly opt in
+- `DESTROY_LOCAL_STATE=false` keeps the last kubeconfigs and generated manifests for inspection
+- `DESTROY_FORGEJO_REPO=false` keeps the remote Git repository unless you explicitly opt in
+
+If any optional cleanup step is enabled but cannot run because credentials are missing, `destroy.sh` prints a skip message describing what was not removed.
+If DNS cleanup or Forgejo repo deletion fails after Terraform teardown, rerun the same cleanup flags or remove the remaining resources manually.
 
 ## Current limitations
-- automated repo creation currently assumes `FORGEJO_REPO_OWNER == FORGEJO_ADMIN_USERNAME`
+- organization-owned repo bootstrap works only when `FORGEJO_REPO_OWNER` names a pre-existing organization that the configured admin can create repositories in; bootstrap does not create the organization itself
-- bootstrap still uses local `docker` to generate the registry htpasswd secret
+- unattended repo seeding now uses an authenticated HTTPS remote built from the configured Forgejo admin credentials, so operators should replace that local remote with a token, SSH, or credential-helper-backed remote after bootstrap if they do not want credentials stored in `.git/config`
+- cloud-init no longer clones a bootstrap repository onto the node; Kubernetes asset delivery is still workstation-driven after Terraform
+- `bootstrap_repo_path` in Terraform is only a reserved marker for a future node-local bootstrap/GitOps flow
+- bootstrap requires either a local `htpasswd` binary or local `docker` as a fallback to generate the registry htpasswd secret
 - bootstrap and CI authentication paths should still be hardened before production use
+- runner identity is persisted under the shared `forgejo-data` PVC, so deleting the `forgejo-runner` pod is safe but deleting that PVC forces re-registration on the next bootstrap run

docs/hetzner-self-hosted-ci-runbook.md

@@ -8,7 +8,7 @@ From your workstation:
 ```bash
 cp scripts/hetzner/bootstrap-secrets.env.example scripts/hetzner/bootstrap-secrets.env
 source scripts/hetzner/bootstrap-secrets.env
-python3 -c 'import nacl'  # verify PyNaCl is installed for Actions secret encryption
+python3 -c 'import yaml, nacl'  # verify PyYAML and PyNaCl are installed for forgejo-actions bootstrap
 bash scripts/hetzner/bootstrap.sh
 ```
 
@@ -24,7 +24,12 @@ After that you should have:
 - Registry reachable at `https://${REGISTRY_DOMAIN}`
 - private admin/control-plane access over Tailscale if configured
 
-Bootstrap repo automation requires `FORGEJO_ADMIN_USERNAME`, `FORGEJO_ADMIN_PASSWORD`, and Python `PyNaCl` locally so the script can encrypt Forgejo Actions secrets before upload. The same bootstrap flow now also creates the initial Forgejo admin account and generates the one-time runner registration token after Forgejo is up.
+Bootstrap repo automation requires `FORGEJO_ADMIN_USERNAME`, `FORGEJO_ADMIN_PASSWORD`, Python `PyYAML` locally for kubeconfig rendering, and Python `PyNaCl` locally in the default `forgejo-actions` mode so the script can encrypt Forgejo Actions secrets before upload. Bootstrap now fails fast with an explicit preflight error if those Python modules are missing. The same bootstrap flow now also creates the initial Forgejo admin account and writes a durable `/data/.runner` config into the shared Forgejo PVC before the runner deployment is allowed to start.
+
+Repository bootstrap is now owner-aware:
+- if `FORGEJO_REPO_OWNER` matches `FORGEJO_ADMIN_USERNAME`, bootstrap creates the repo under the admin user's namespace
+- if `FORGEJO_REPO_OWNER` names an existing Forgejo organization that the admin can manage, bootstrap creates the repo under that organization instead
+- rerunning bootstrap remains idempotent because repo creation is skipped when the target repo already exists and secrets/variables are upserted in place
 
 ## Verify the cluster
 ```bash
@@ -37,7 +42,7 @@ kubectl -n unrip get deploy,pods
 ```
 
 ## Seed the repo into Forgejo
-Default bootstrap already seeds the repo with:
+Default bootstrap already seeds the repo with HTTPS Git auth derived from the configured admin credentials:
 
 ```bash
 bash scripts/hetzner/seed-forgejo-repo.sh
@@ -45,6 +50,8 @@ bash scripts/hetzner/seed-forgejo-repo.sh
 
 You only need to run it manually if you skipped seeding during bootstrap or want to push again after local changes.
 
+`seed-forgejo-repo.sh` rewrites the configured `forgejo` remote to an authenticated HTTPS URL for non-interactive pushes. That makes bootstrap reruns hands-free, but it also means the local Git remote can contain embedded credentials. Replace it with a token-backed, SSH, or credential-helper-managed remote after bootstrap if you do not want secrets persisted in `.git/config`.
+
 ## Configure Forgejo Actions secrets and variables
 Bootstrap upserts these repository secrets automatically:
 - `KUBECONFIG_B64`
@@ -65,7 +72,7 @@ The workflow in `.forgejo/workflows/deploy.yml` now:
 2. loads kubeconfig from `KUBECONFIG_B64`
 3. computes `IMAGE=${REGISTRY_HOST}/${PROJECT_NAME}:${GIT_SHA}`
 4. creates an in-cluster Kubernetes Job in `PROJECT_NAMESPACE`
-5. that Job checks out the repo with the Forgejo job token in an init container
+5. that Job checks out the repo with the Forgejo job token in an init container using an `Authorization: Bearer ...` header instead of embedding the token in the clone URL
 6. Kaniko builds and pushes the image using the Kubernetes registry auth secret
 7. the workflow updates each deployment listed in `PROJECT_DEPLOYMENTS` inside `PROJECT_NAMESPACE`
 8. the workflow waits for rollout after each image update
@@ -91,6 +98,8 @@ Legacy mode still exists if you explicitly set:
 BOOTSTRAP_DELIVERY_MODE=local-image-import
 ```
 
+That legacy mode still requires local `docker` to build and import the bootstrap image. In all modes, bootstrap also needs either a native `htpasswd` binary or local `docker` as a fallback to generate the registry auth secret.
+
 ## Trigger deploys
 Push to `main` in Forgejo:
 
@@ -116,10 +125,15 @@ Currently supported DNS providers:
 - Cloudflare
 - Porkbun
 
+Destroy does not remove those external DNS records unless you explicitly opt in with `DESTROY_DNS=true` when running `scripts/hetzner/destroy.sh`.
+Likewise, generated local kubeconfigs/manifests remain on disk unless you set `DESTROY_LOCAL_STATE=true`, and the seeded Forgejo repository remains unless you set `DESTROY_FORGEJO_REPO=true` with valid Forgejo admin credentials.
+
 TLS is issued by cert-manager using the rendered Let's Encrypt email and ingress hosts.
 
 ## Current limitations
-- the bootstrap path now creates the initial admin account and one-time runner registration token automatically from inside the Forgejo pod, but it still depends on the operator supplying the intended admin credentials up front
+- the bootstrap path now creates the initial admin account and runner config automatically from inside the Forgejo pod, but it still depends on the operator supplying the intended admin credentials up front
-- runner registration no longer needs a pre-seeded Kubernetes secret, but the runner config still lives on `emptyDir`, so bootstrap must recreate `/data/.runner` after a runner pod replacement
+- runner startup is now manifest-gated on a durable `/data/.runner` file stored under the shared `forgejo-data` PVC, so fresh applies no longer depend on a broken intermediate secret or a race against a crashing runner pod; deleting that Forgejo PVC still requires rerunning bootstrap to re-register the runner
-- automated repo creation currently assumes `FORGEJO_REPO_OWNER == FORGEJO_ADMIN_USERNAME`
+- organization-owned repo bootstrap works only when `FORGEJO_REPO_OWNER` names a pre-existing organization that the configured admin can create repositories in; bootstrap does not create the organization itself
+- `seed-forgejo-repo.sh` uses admin-password-backed HTTPS pushes by default for unattended bootstrap, so operators should swap to a token or SSH remote after initial seeding if they want to avoid storing credentials in `.git/config`
+- `destroy.sh` can now remove the seeded Forgejo repository, DNS records, and local bootstrap artifacts, but each destructive cleanup path is opt-in via `DESTROY_FORGEJO_REPO=true`, `DESTROY_DNS=true`, and `DESTROY_LOCAL_STATE=true`
 - the runner currently uses host-mode jobs and installs `kubectl` at job start; the image build itself runs in-cluster via Kaniko, which is functional but not yet optimized

scripts/hetzner/bootstrap-secrets.env.example

@@ -7,6 +7,18 @@
 # - set *_PASS variables to pass entry paths
 # - optionally override a value directly via ENV for CI/tests or one-off debugging
 # - bootstrap/destroy will load the first line from each pass entry without echoing it
+#
+# What bootstrap materializes from this file:
+# - overwrites deploy/k8s/overlays/hetzner-single-node/secrets/unrip.env
+# - overwrites deploy/k8s/overlays/hetzner-single-node/secrets/forgejo.env
+# - renders generated ingress/issuer patches under .state/hetzner/generated-overlay/
+# - creates registry-secrets and the project docker-registry pull secret imperatively
+#
+# Checked-in overlay note:
+# - plain `kubectl apply -k deploy/k8s/overlays/hetzner-single-node` only uses the
+#   checked-in files under deploy/k8s/overlays/hetzner-single-node/
+# - it does not read this file automatically
+# - it does not create registry auth secrets for you
 
 export PASS_PREFIX="infra/unrip3"
 pass_ref() {
@@ -46,6 +58,7 @@ export LETSENCRYPT_EMAIL="${LETSENCRYPT_EMAIL:-ops@example.com}"
 
 # Optional DNS automation: choose one provider
 # Cloudflare
+# bootstrap/destroy auto-resolve both values from pass when *_PASS is set.
 export CLOUDFLARE_API_TOKEN_PASS="${CLOUDFLARE_API_TOKEN_PASS:-$(pass_ref cloudflare/api-token)}"
 export CLOUDFLARE_ZONE_ID_PASS="${CLOUDFLARE_ZONE_ID_PASS:-$(pass_ref cloudflare/zone-id)}"
 # Porkbun

scripts/hetzner/bootstrap.sh

@@ -21,8 +21,10 @@ require curl
 require python3
 require ssh
 require git
-require docker
 require base64
+require realpath
+
+require_python_modules python3 yaml
 
 resolve_secret_var HCLOUD_TOKEN required
 resolve_secret_var TAILSCALE_AUTH_KEY optional
@@ -30,6 +32,7 @@ resolve_secret_var NEAR_INTENTS_API_KEY required
 resolve_secret_var REGISTRY_PASSWORD required
 resolve_secret_var FORGEJO_ADMIN_PASSWORD required
 resolve_secret_var CLOUDFLARE_API_TOKEN optional
+resolve_secret_var CLOUDFLARE_ZONE_ID optional
 resolve_secret_var PORKBUN_API_KEY optional
 resolve_secret_var PORKBUN_SECRET_API_KEY optional
 
@@ -39,9 +42,12 @@ resolve_secret_var PORKBUN_SECRET_API_KEY optional
 : "${BASE_DOMAIN:?set BASE_DOMAIN}"
 : "${FORGEJO_DOMAIN:=git.${BASE_DOMAIN}}"
 : "${FORGEJO_ROOT_URL:=https://${FORGEJO_DOMAIN}/}"
+: "${FORGEJO_INTERNAL_URL:=http://forgejo.forgejo.svc.cluster.local:3000/}"
 : "${REGISTRY_DOMAIN:=registry.${BASE_DOMAIN}}"
 : "${REGISTRY_USERNAME:?set REGISTRY_USERNAME}"
+: "${TAILSCALE_CONTROL_PLANE_HOSTNAME:=}"
 : "${TF_ADMIN_CIDR_BLOCKS:=}"
+: "${BOOTSTRAP_ALLOW_PUBLIC_ADMIN_FALLBACK:=1}"
 : "${PROJECT_NAME:=$DEFAULT_PROJECT_NAME}"
 : "${PROJECT_NAMESPACE:=$DEFAULT_PROJECT_NAMESPACE}"
 : "${PROJECT_OVERLAY_DIR:=$OVERLAY_DIR}"
@@ -65,6 +71,18 @@ resolve_secret_var PORKBUN_SECRET_API_KEY optional
 BOOTSTRAP_IMAGE="${PROJECT_IMAGE_REPOSITORY}:bootstrap"
 PROJECT_SECRET_ENV_PATH="$PROJECT_OVERLAY_DIR/secrets/$PROJECT_SECRET_ENV_BASENAME"
 GENERATED_OVERLAY_DIR="$STATE_DIR/generated-overlay"
+
+if [[ "$BOOTSTRAP_DELIVERY_MODE" != "forgejo-actions" ]]; then
+  require docker
+fi
+if [[ -n "${TAILSCALE_AUTH_KEY:-}" && "$TF_ADMIN_CIDR_BLOCKS" == '[]' && "$BOOTSTRAP_ALLOW_PUBLIC_ADMIN_FALLBACK" == "1" ]]; then
+  OPERATOR_PUBLIC_IP="$(curl -fsS https://api.ipify.org || true)"
+  if [[ "$OPERATOR_PUBLIC_IP" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    TF_ADMIN_CIDR_BLOCKS="[\"${OPERATOR_PUBLIC_IP}/32\"]"
+    echo "tailscale bootstrap fallback enabled for operator public IP ${OPERATOR_PUBLIC_IP}/32"
+  fi
+fi
+
 SSH_PUBLIC_KEY=$(cat "$SSH_PUBLIC_KEY_PATH")
 SSH_PRIVATE_KEY_PATH="${SSH_PUBLIC_KEY_PATH%.pub}"
 if [[ ! -f "$SSH_PRIVATE_KEY_PATH" ]]; then
@@ -76,7 +94,6 @@ TF_VARS=(
   -var "hcloud_token=$HCLOUD_TOKEN"
   -var "ssh_public_key=$SSH_PUBLIC_KEY"
   -var "public_domain=$PUBLIC_DOMAIN"
-  -var "bootstrap_repo_url=local-bootstrap"
   -var "tailscale_auth_key=${TAILSCALE_AUTH_KEY:-}"
   -var "tailscale_control_plane_hostname=${TAILSCALE_CONTROL_PLANE_HOSTNAME:-}"
 )
@@ -95,12 +112,19 @@ fi
 
 SERVER_IP=$(terraform -chdir="$TF_DIR" output -raw server_ipv4)
 K3S_API_URL=$(terraform -chdir="$TF_DIR" output -raw k3s_api_url)
-if [[ -n "${TAILSCALE_AUTH_KEY:-}" ]]; then
-  DISCOVERED_TAILSCALE_HOST="${TAILSCALE_CONTROL_PLANE_HOSTNAME:-$(wait_for_tailscale_node "$BOOTSTRAP_NODE_NAME")}" 
-  SSH_TARGET="root@${DISCOVERED_TAILSCALE_HOST}"
-  K3S_API_URL="https://${DISCOVERED_TAILSCALE_HOST}:6443"
-else
 SSH_TARGET="root@${SERVER_IP}"
+USE_SSH_TUNNEL_FOR_K3S=0
+if [[ -n "${TAILSCALE_AUTH_KEY:-}" ]]; then
+  DISCOVERED_TAILSCALE_HOST="$(wait_for_tailscale_node "$BOOTSTRAP_NODE_NAME" 24 5 || true)"
+  if [[ -n "$DISCOVERED_TAILSCALE_HOST" ]]; then
+    SSH_TARGET="root@${DISCOVERED_TAILSCALE_HOST}"
+    USE_SSH_TUNNEL_FOR_K3S=1
+  elif [[ "$TF_ADMIN_CIDR_BLOCKS" != '[]' ]]; then
+    echo "tailscale node did not appear in time; falling back to public admin access for this bootstrap run" >&2
+  else
+    echo "tailscale node did not appear in time and no public admin fallback is configured" >&2
+    exit 1
+  fi
 fi
 
 if [[ -n "${CLOUDFLARE_API_TOKEN:-}" && -n "${CLOUDFLARE_ZONE_ID:-}" ]]; then
@@ -114,12 +138,33 @@ elif [[ -n "${PORKBUN_API_KEY:-}" && -n "${PORKBUN_SECRET_API_KEY:-}" ]]; then
 fi
 
 wait_for_ssh "$SSH_TARGET"
-echo "waiting for Kubernetes API on $K3S_API_URL..."
+if [[ "$USE_SSH_TUNNEL_FOR_K3S" == "1" ]]; then
-wait_for_url "${K3S_API_URL}/readyz" "k3s API readiness"
+  LOCAL_K3S_TUNNEL_PORT="${LOCAL_K3S_TUNNEL_PORT:-$(python3 - <<'PY'
+import socket
+s=socket.socket()
+s.bind(('127.0.0.1', 0))
+print(s.getsockname()[1])
+s.close()
+PY
+)}"
+  K3S_API_URL="https://localhost:${LOCAL_K3S_TUNNEL_PORT}"
+  ssh -i "$SSH_PRIVATE_KEY_PATH" \
+    -o StrictHostKeyChecking=no \
+    -o UserKnownHostsFile=/dev/null \
+    -o ExitOnForwardFailure=yes \
+    -o ServerAliveInterval=30 \
+    -o ServerAliveCountMax=3 \
+    -N -L "127.0.0.1:${LOCAL_K3S_TUNNEL_PORT}:127.0.0.1:6443" \
+    "$SSH_TARGET" >/dev/null 2>&1 &
+  K3S_TUNNEL_PID=$!
+  trap 'if [[ -n "${K3S_TUNNEL_PID:-}" ]]; then kill "$K3S_TUNNEL_PID" >/dev/null 2>&1 || true; fi' EXIT
+fi
 
+echo "waiting for Kubernetes API on $K3S_API_URL..."
 ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$SSH_TARGET" 'sudo cat /etc/rancher/k3s/k3s.yaml' \
   | sed "s|https://127.0.0.1:6443|${K3S_API_URL}|" > "$KUBECONFIG_PATH"
 export KUBECONFIG="$KUBECONFIG_PATH"
+wait_for_kubectl 120 5
 
 python3 - "$KUBECONFIG_PATH" "$CI_KUBECONFIG_PATH" <<'PY'
 import sys
@@ -139,21 +184,39 @@ root_url=$FORGEJO_ROOT_URL
 domain=$FORGEJO_DOMAIN
 EOF
 python3 - <<PY
+import os
 from pathlib import Path
 
-root = Path("$PROJECT_OVERLAY_DIR")
+root = Path("$PROJECT_OVERLAY_DIR").resolve()
-generated_root = Path("$GENERATED_OVERLAY_DIR")
+generated_root = Path("$GENERATED_OVERLAY_DIR").resolve()
 project_kustomize_path = "$PROJECT_KUSTOMIZE_PATH"
 project_namespace = "$PROJECT_NAMESPACE"
 project_secret_name = "$PROJECT_SECRET_NAME"
 project_secret_env_basename = "$PROJECT_SECRET_ENV_BASENAME"
-project_overlay_dir = Path("$PROJECT_OVERLAY_DIR").relative_to(Path("$ROOT_DIR"))
+platform_base = (root / "../../platform/base").resolve()
+project_base = (root / project_kustomize_path).resolve() if project_kustomize_path else None
+project_secret_env = (root / "secrets" / project_secret_env_basename).resolve()
+forgejo_secret_env = (root / "secrets" / "forgejo.env").resolve()
+platform_resources = [
+    platform_base / "namespace.yaml",
+    platform_base / "forgejo.yaml",
+    platform_base / "forgejo-rbac.yaml",
+    platform_base / "forgejo-runner.yaml",
+    platform_base / "registry.yaml",
+    platform_base / "ingress.yaml",
+    platform_base / "cluster-issuers.yaml",
+    platform_base / "coredns.yaml",
+]
 
-resources = [f"../../{project_overlay_dir}/../../platform/base"]
+resources = [os.path.relpath(path, generated_root) for path in platform_resources]
-if project_kustomize_path:
+if project_base:
-    resources.append(f"../../{project_overlay_dir}/{project_kustomize_path}")
+    resources.append(os.path.relpath(project_base, generated_root))
 
 generated_root.mkdir(parents=True, exist_ok=True)
+project_secret_env_rel = Path(project_secret_env.name)
+forgejo_secret_env_rel = Path(forgejo_secret_env.name)
+(generated_root / project_secret_env_rel).write_text(project_secret_env.read_text())
+(generated_root / forgejo_secret_env_rel).write_text(forgejo_secret_env.read_text())
 (generated_root / "kustomization.yaml").write_text(
     """apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
@@ -168,22 +231,18 @@ secretGenerator:
   - name: {project_secret_name}
     namespace: {project_namespace}
     envs:
-      - ../../{project_overlay_dir}/secrets/{project_secret_env_basename}
+      - {project_secret_env_rel}
   - name: forgejo-secrets
     namespace: forgejo
     envs:
-      - ../../{project_overlay_dir}/secrets/forgejo.env
+      - {forgejo_secret_env_rel}
-  - name: registry-secrets
-    namespace: registry
-    files:
-      - htpasswd=../../{project_overlay_dir}/secrets/registry.htpasswd
 generatorOptions:
   disableNameSuffixHash: true
 """.format(
         project_secret_name=project_secret_name,
         project_namespace=project_namespace,
-        project_overlay_dir=project_overlay_dir,
+        project_secret_env_rel=project_secret_env_rel,
-        project_secret_env_basename=project_secret_env_basename,
+        forgejo_secret_env_rel=forgejo_secret_env_rel,
     )
 )
 (generated_root / "storage-class.patch.yaml").write_text((root / "storage-class.patch.yaml").read_text())
@@ -194,19 +253,23 @@ PY
 kubectl apply -f "$ROOT_DIR/deploy/k8s/platform/base/namespace.yaml"
 kubectl create namespace "$PROJECT_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f -
 kubectl -n registry create secret generic registry-secrets \
-  --from-file=htpasswd=<(docker run --rm --entrypoint htpasswd httpd:2 -Bbn "$REGISTRY_USERNAME" "$REGISTRY_PASSWORD") \
+  --from-file=htpasswd=<(generate_htpasswd "$REGISTRY_USERNAME" "$REGISTRY_PASSWORD") \
   --dry-run=client -o yaml | kubectl apply -f -
 kubectl -n "$PROJECT_NAMESPACE" create secret docker-registry "$PROJECT_REGISTRY_SECRET_NAME" \
   --docker-server="$REGISTRY_DOMAIN" \
   --docker-username="$REGISTRY_USERNAME" \
   --docker-password="$REGISTRY_PASSWORD" \
   --dry-run=client -o yaml | kubectl apply -f -
-kubectl apply -k "$GENERATED_OVERLAY_DIR"
+
+kubectl -n cert-manager delete deployment cert-manager cert-manager-webhook cert-manager-cainjector --ignore-not-found --wait=true || true
+kubectl apply -f "$ROOT_DIR/deploy/k8s/platform/base/cert-manager.yaml"
+kubectl wait --for=condition=Established --timeout=180s crd/certificates.cert-manager.io
+kubectl wait --for=condition=Established --timeout=180s crd/clusterissuers.cert-manager.io
+kubectl apply -k "$PROJECT_OVERLAY_DIR"
 
 kubectl -n forgejo rollout status deployment/forgejo --timeout=300s
 kubectl -n registry rollout status deployment/registry --timeout=300s
 kubectl -n "$PROJECT_NAMESPACE" rollout status deployment/redpanda --timeout=300s
-kubectl -n forgejo wait --for=condition=available deployment/forgejo-runner --timeout=300s || true
 
 forgejo_admin_user_b64=$(printf '%s' "$FORGEJO_ADMIN_USERNAME" | base64 | tr -d '\n')
 forgejo_admin_pass_b64=$(printf '%s' "$FORGEJO_ADMIN_PASSWORD" | base64 | tr -d '\n')
@@ -218,39 +281,78 @@ set -euo pipefail
 ADMIN_USERNAME=\$(printf '%s' '$forgejo_admin_user_b64' | base64 -d)
 ADMIN_PASSWORD=\$(printf '%s' '$forgejo_admin_pass_b64' | base64 -d)
 ADMIN_EMAIL=\$(printf '%s' '$forgejo_admin_email_b64' | base64 -d)
-RUNNER_NAME=\$(printf '%s' '$forgejo_runner_name_b64' | base64 -d)
-RUNNER_LABELS=\$(printf '%s' '$forgejo_runner_labels_b64' | base64 -d)
 APP_INI=/data/gitea/conf/app.ini
-if [[ ! -f "$APP_INI" ]]; then
+if [[ ! -f "\$APP_INI" ]]; then
-  echo "missing Forgejo config at $APP_INI" >&2
+  echo "missing Forgejo config at \$APP_INI" >&2
   exit 1
 fi
 if su-exec git /usr/local/bin/forgejo admin user list | awk 'NR>1 {print \$2}' | grep -qx "\$ADMIN_USERNAME"; then
   echo "forgejo admin already exists: \$ADMIN_USERNAME"
 else
-  su-exec git /usr/local/bin/forgejo admin user create --config "$APP_INI" --admin --username "\$ADMIN_USERNAME" --password "\$ADMIN_PASSWORD" --email "\$ADMIN_EMAIL" --must-change-password=false
+  su-exec git /usr/local/bin/forgejo admin user create --config "\$APP_INI" --admin --username "\$ADMIN_USERNAME" --password "\$ADMIN_PASSWORD" --email "\$ADMIN_EMAIL" --must-change-password=false
-fi
-if [[ ! -f /data/.runner ]]; then
-  RUNNER_TOKEN=\$(su-exec git /usr/local/bin/forgejo --config "$APP_INI" actions generate-runner-token)
-  forgejo-runner register --no-interactive --name "\$RUNNER_NAME" --instance "$FORGEJO_ROOT_URL" --token "\$RUNNER_TOKEN" --labels "\$RUNNER_LABELS"
-  install -o 1000 -g 1000 -m 600 .runner /data/.runner
-  rm -f .runner
-  echo "registered forgejo runner config at /data/.runner"
-else
-  echo "forgejo runner already configured: /data/.runner"
 fi
 EOF
+RUNNER_NAME="$(printf '%s' "$forgejo_runner_name_b64" | base64 -d)"
+RUNNER_LABELS="$(printf '%s' "$forgejo_runner_labels_b64" | base64 -d)"
+RUNNER_TOKEN="$(kubectl -n forgejo exec deploy/forgejo -- /bin/bash --noprofile --norc -lc 'su-exec git /usr/local/bin/forgejo --config /data/gitea/conf/app.ini actions generate-runner-token' | tr -d '\r\n')"
+kubectl -n forgejo delete job forgejo-runner-bootstrap --ignore-not-found
+cat <<EOF | kubectl apply -f -
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: forgejo-runner-bootstrap
+  namespace: forgejo
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      restartPolicy: Never
+      volumes:
+        - name: forgejo-data
+          persistentVolumeClaim:
+            claimName: forgejo-data
+      containers:
+        - name: register
+          image: code.forgejo.org/forgejo/runner:6.3.1
+          command: ["/bin/sh", "-lc"]
+          args:
+            - >-
+              mkdir -p /data &&
+              if [ ! -s /data/.runner ]; then
+                forgejo-runner register --no-interactive --name "$RUNNER_NAME" --instance "$FORGEJO_INTERNAL_URL" --token "$RUNNER_TOKEN" --labels "$RUNNER_LABELS";
+              else
+                echo runner config already exists;
+              fi
+          volumeMounts:
+            - name: forgejo-data
+              mountPath: /data
+              subPath: forgejo-runner
+EOF
+kubectl -n forgejo wait --for=condition=complete --timeout=300s job/forgejo-runner-bootstrap
 kubectl -n forgejo rollout restart deployment/forgejo-runner
 kubectl -n forgejo rollout status deployment/forgejo-runner --timeout=300s
 
-if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then
+FORGEJO_BOOTSTRAP_PORT="${FORGEJO_BOOTSTRAP_PORT:-$(python3 - <<'PY'
-  wait_for_url "$FORGEJO_ROOT_URL" "Forgejo public URL" 180 5
+import socket
-  wait_for_http_status "https://$REGISTRY_DOMAIN/v2/" "registry public URL" '200|401' 180 5
+s=socket.socket()
+s.bind(('127.0.0.1', 0))
+print(s.getsockname()[1])
+s.close()
+PY
+)}"
+FORGEJO_BOOTSTRAP_URL="http://127.0.0.1:${FORGEJO_BOOTSTRAP_PORT}"
+kubectl -n forgejo port-forward svc/forgejo "${FORGEJO_BOOTSTRAP_PORT}:3000" >/tmp/forgejo-port-forward.log 2>&1 &
+FORGEJO_PORT_FORWARD_PID=$!
+trap 'if [[ -n "${FORGEJO_PORT_FORWARD_PID:-}" ]]; then kill "$FORGEJO_PORT_FORWARD_PID" >/dev/null 2>&1 || true; fi; if [[ -n "${K3S_TUNNEL_PID:-}" ]]; then kill "$K3S_TUNNEL_PID" >/dev/null 2>&1 || true; fi' EXIT
+wait_for_url "$FORGEJO_BOOTSTRAP_URL" "Forgejo bootstrap URL" 60 2
 
+if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then
+  FORGEJO_ADMIN_API_TOKEN="$(kubectl -n forgejo exec deploy/forgejo -- /bin/bash --noprofile --norc -lc "su-exec git /usr/local/bin/forgejo admin user generate-access-token --config /data/gitea/conf/app.ini --username '$FORGEJO_ADMIN_USERNAME' --token-name bootstrap-$(date +%s) --scopes read:user,read:repository,write:repository,write:user --raw" | tr -d '\r\n')"
   forgejo_bootstrap_args=(
-    --forgejo-url "$FORGEJO_ROOT_URL"
+    --forgejo-url "$FORGEJO_BOOTSTRAP_URL"
+    --token "$FORGEJO_ADMIN_API_TOKEN"
     --admin-username "$FORGEJO_ADMIN_USERNAME"
-    --admin-password "$FORGEJO_ADMIN_PASSWORD"
     --repo-owner "$FORGEJO_REPO_OWNER"
     --repo-name "$FORGEJO_REPO_NAME"
     --kubeconfig "$KUBECONFIG_PATH"
@@ -266,7 +368,10 @@ if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then
   fi
   python3 "$ROOT_DIR/scripts/hetzner/forgejo-bootstrap.py" "${forgejo_bootstrap_args[@]}"
 
-  bash "$ROOT_DIR/scripts/hetzner/seed-forgejo-repo.sh"
+  FORGEJO_PUSH_URL_BASE="$FORGEJO_BOOTSTRAP_URL" bash "$ROOT_DIR/scripts/hetzner/seed-forgejo-repo.sh"
+
+  wait_for_url "$FORGEJO_ROOT_URL" "Forgejo public URL" 180 5
+  wait_for_http_status "https://$REGISTRY_DOMAIN/v2/" "registry public URL" '200|401' 180 5
 else
   docker build -t "$BOOTSTRAP_IMAGE" "$ROOT_DIR"
   docker save "$BOOTSTRAP_IMAGE" \

scripts/hetzner/destroy.sh

@@ -7,22 +7,38 @@ source "$ROOT_DIR/scripts/hetzner/lib.sh"
 load_bootstrap_env
 
 TF_DIR="$ROOT_DIR/infra/terraform/hetzner"
+STATE_DIR="$ROOT_DIR/.state/hetzner"
+GENERATED_OVERLAY_DIR="$ROOT_DIR/deploy/k8s/overlays/hetzner-single-node/generated"
+DESTROY_DNS="${DESTROY_DNS:-false}"
+DESTROY_LOCAL_STATE="${DESTROY_LOCAL_STATE:-false}"
+DESTROY_FORGEJO_REPO="${DESTROY_FORGEJO_REPO:-false}"
 
 require terraform
+require curl
 
 resolve_secret_var HCLOUD_TOKEN required
 resolve_secret_var TAILSCALE_AUTH_KEY optional
+resolve_secret_var CLOUDFLARE_API_TOKEN optional
+resolve_secret_var CLOUDFLARE_ZONE_ID optional
+resolve_secret_var PORKBUN_API_KEY optional
+resolve_secret_var PORKBUN_SECRET_API_KEY optional
+resolve_secret_var FORGEJO_ADMIN_PASSWORD optional
 
 : "${SSH_PUBLIC_KEY_PATH:?set SSH_PUBLIC_KEY_PATH}"
 : "${PUBLIC_DOMAIN:=bootstrap.example.com}"
+: "${BASE_DOMAIN:?set BASE_DOMAIN}"
 : "${TAILSCALE_CONTROL_PLANE_HOSTNAME:=}"
 : "${TF_ADMIN_CIDR_BLOCKS:=}"
+: "${FORGEJO_DOMAIN:=}"
+: "${FORGEJO_REPO_OWNER:=${FORGEJO_ADMIN_USERNAME:-}}"
+: "${FORGEJO_REPO_NAME:=unrip}"
 
 SSH_PUBLIC_KEY=$(cat "$SSH_PUBLIC_KEY_PATH")
 TF_VARS=(
   -var "hcloud_token=$HCLOUD_TOKEN"
   -var "ssh_public_key=$SSH_PUBLIC_KEY"
   -var "public_domain=$PUBLIC_DOMAIN"
+  -var "bootstrap_repo_url=local-bootstrap"
   -var "tailscale_auth_key=${TAILSCALE_AUTH_KEY:-}"
   -var "tailscale_control_plane_hostname=$TAILSCALE_CONTROL_PLANE_HOSTNAME"
 )
@@ -33,3 +49,63 @@ fi
 
 terraform -chdir="$TF_DIR" init
 terraform -chdir="$TF_DIR" destroy -auto-approve "${TF_VARS[@]}"
+
+cleanup_dns() {
+  if [[ "$DESTROY_DNS" != "true" ]]; then
+    echo "skipping DNS cleanup (set DESTROY_DNS=true to remove bootstrap-managed records)"
+    return 0
+  fi
+
+  if [[ -n "${CLOUDFLARE_API_TOKEN:-}" && -n "${CLOUDFLARE_ZONE_ID:-}" ]]; then
+    DNS_MODE=delete BASE_DOMAIN="$BASE_DOMAIN" bash "$ROOT_DIR/scripts/hetzner/configure-cloudflare-dns.sh"
+  elif [[ -n "${PORKBUN_API_KEY:-}" && -n "${PORKBUN_SECRET_API_KEY:-}" ]]; then
+    DNS_MODE=delete BASE_DOMAIN="$BASE_DOMAIN" bash "$ROOT_DIR/scripts/hetzner/configure-porkbun-dns.sh"
+  else
+    echo "skipping DNS cleanup (no supported DNS provider credentials available)"
+  fi
+}
+
+cleanup_local_state() {
+  if [[ "$DESTROY_LOCAL_STATE" != "true" ]]; then
+    echo "skipping local artifact cleanup (set DESTROY_LOCAL_STATE=true to remove generated bootstrap outputs)"
+    return 0
+  fi
+
+  rm -rf "$STATE_DIR" "$GENERATED_OVERLAY_DIR"
+  rm -f "$TF_DIR/terraform.tfstate" "$TF_DIR/terraform.tfstate.backup"
+  rm -rf "$TF_DIR/.terraform"
+  echo "removed local bootstrap artifacts from .state/hetzner, generated overlay outputs, and Terraform working state"
+}
+
+cleanup_forgejo_repo() {
+  if [[ "$DESTROY_FORGEJO_REPO" != "true" ]]; then
+    echo "skipping Forgejo repo cleanup (set DESTROY_FORGEJO_REPO=true to delete the bootstrap-managed repo)"
+    return 0
+  fi
+
+  if [[ -z "$FORGEJO_DOMAIN" || -z "${FORGEJO_ADMIN_USERNAME:-}" || -z "${FORGEJO_ADMIN_PASSWORD:-}" || -z "$FORGEJO_REPO_OWNER" || -z "$FORGEJO_REPO_NAME" ]]; then
+    echo "skipping Forgejo repo cleanup (set FORGEJO_DOMAIN, FORGEJO_ADMIN_USERNAME, FORGEJO_ADMIN_PASSWORD, FORGEJO_REPO_OWNER, and FORGEJO_REPO_NAME)"
+    return 0
+  fi
+
+  local base_url="https://${FORGEJO_DOMAIN}"
+  local status
+  status=$(curl -ksS -o /dev/null -w '%{http_code}' -u "$FORGEJO_ADMIN_USERNAME:$FORGEJO_ADMIN_PASSWORD" \
+    -X DELETE "$base_url/api/v1/repos/$FORGEJO_REPO_OWNER/$FORGEJO_REPO_NAME")
+
+  case "$status" in
+    204)
+      echo "deleted Forgejo repo $FORGEJO_REPO_OWNER/$FORGEJO_REPO_NAME"
+      ;;
+    404)
+      echo "skipped missing Forgejo repo $FORGEJO_REPO_OWNER/$FORGEJO_REPO_NAME"
+      ;;
+    *)
+      echo "warning: Forgejo repo cleanup returned HTTP $status for $FORGEJO_REPO_OWNER/$FORGEJO_REPO_NAME" >&2
+      ;;
+  esac
+}
+
+cleanup_dns
+cleanup_local_state
+cleanup_forgejo_repo

scripts/hetzner/forgejo-bootstrap.py

@@ -10,18 +10,23 @@ import urllib.request
 from pathlib import Path
 
 import yaml
-from nacl import encoding, public
 
 
 class ForgejoClient:
-    def __init__(self, base_url: str, username: str, password: str):
+    def __init__(self, base_url: str, username: str | None = None, password: str | None = None, token: str | None = None):
         self.base_url = base_url.rstrip('/')
-        credentials = base64.b64encode(f'{username}:{password}'.encode()).decode()
+        self.username = username or ''
         self.headers = {
-            'Authorization': f'Basic {credentials}',
             'Accept': 'application/json',
             'Content-Type': 'application/json',
         }
+        if token:
+            self.headers['Authorization'] = f'token {token}'
+        elif username is not None and password is not None:
+            credentials = base64.b64encode(f'{username}:{password}'.encode()).decode()
+            self.headers['Authorization'] = f'Basic {credentials}'
+        else:
+            raise ValueError('ForgejoClient requires either token auth or username/password auth')
         self.ssl_context = ssl.create_default_context()
 
     def request(self, method: str, path: str, payload=None, expected=(200, 201, 204)):
@@ -54,19 +59,22 @@ class ForgejoClient:
                 return None
             raise
 
-    def create_repo(self, name: str, private: bool):
+    def create_repo(self, owner: str, name: str, private: bool):
-        return self.request('POST', '/api/v1/user/repos', {
+        payload = {
             'name': name,
             'private': private,
             'auto_init': False,
             'default_branch': 'main',
-        }, expected=(201,))
+        }
+        if owner == self.username:
+            return self.request('POST', '/api/v1/user/repos', payload, expected=(201,))
+        return self.request('POST', f'/api/v1/orgs/{urllib.parse.quote(owner)}/repos', payload, expected=(201,))
 
     def upsert_variable(self, owner: str, repo: str, name: str, value: str):
         try:
             self.request('POST', f'/api/v1/repos/{owner}/{repo}/actions/variables/{urllib.parse.quote(name)}', {
                 'value': value,
-            }, expected=(201,))
+            }, expected=(201, 204))
         except RuntimeError as exc:
             if ' returned 409:' not in str(exc) and ' returned 422:' not in str(exc):
                 raise
@@ -75,13 +83,8 @@ class ForgejoClient:
             }, expected=(201, 204))
 
     def upsert_secret(self, owner: str, repo: str, name: str, value: str):
-        key = self.request('GET', f'/api/v1/repos/{owner}/{repo}/actions/secrets/public-key')
-        public_key = public.PublicKey(key['key'].encode(), encoder=encoding.Base64Encoder)
-        sealed_box = public.SealedBox(public_key)
-        encrypted_value = base64.b64encode(sealed_box.encrypt(value.encode())).decode()
         self.request('PUT', f'/api/v1/repos/{owner}/{repo}/actions/secrets/{urllib.parse.quote(name)}', {
-            'encrypted_value': encrypted_value,
+            'data': value,
-            'key_id': key['key_id'],
         }, expected=(201, 204))
 
 
@@ -97,8 +100,9 @@ def render_ci_kubeconfig(source: Path) -> str:
 def main():
     parser = argparse.ArgumentParser(description='Bootstrap Forgejo repo secrets/variables for CI/CD')
     parser.add_argument('--forgejo-url', required=True)
-    parser.add_argument('--admin-username', required=True)
+    parser.add_argument('--admin-username')
-    parser.add_argument('--admin-password', required=True)
+    parser.add_argument('--admin-password')
+    parser.add_argument('--token')
     parser.add_argument('--repo-owner', required=True)
     parser.add_argument('--repo-name', required=True)
     parser.add_argument('--repo-private', action='store_true', default=False)
@@ -111,10 +115,10 @@ def main():
     parser.add_argument('--project-deployments', required=True)
     args = parser.parse_args()
 
-    client = ForgejoClient(args.forgejo_url, args.admin_username, args.admin_password)
+    client = ForgejoClient(args.forgejo_url, args.admin_username, args.admin_password, args.token)
     repo = client.get_repo(args.repo_owner, args.repo_name)
     if repo is None:
-        created = client.create_repo(args.repo_name, args.repo_private)
+        created = client.create_repo(args.repo_owner, args.repo_name, args.repo_private)
         print(f'created repo {created["full_name"]}')
     else:
         print(f'repo already exists: {repo["full_name"]}')

scripts/hetzner/lib.sh

@@ -8,6 +8,54 @@ require() {
   command -v "$1" >/dev/null 2>&1 || { echo "missing command: $1" >&2; exit 1; }
 }
 
+require_python_modules() {
+  local python_bin="${1:-python3}"
+  shift || true
+  local modules=("$@")
+
+  [[ ${#modules[@]} -gt 0 ]] || return 0
+
+  "$python_bin" - "${modules[@]}" <<'PY'
+import importlib
+import sys
+
+missing = []
+for module in sys.argv[1:]:
+    try:
+        importlib.import_module(module)
+    except ModuleNotFoundError:
+        missing.append(module)
+
+if missing:
+    names = ", ".join(missing)
+    print(
+        f"missing Python module(s): {names}. Install them for {sys.executable} before running bootstrap ",
+        f"(for example: {sys.executable} -m pip install {' '.join(missing)})",
+        sep="",
+        file=sys.stderr,
+    )
+    raise SystemExit(1)
+PY
+}
+
+generate_htpasswd() {
+  local username="$1"
+  local password="$2"
+
+  if command -v htpasswd >/dev/null 2>&1; then
+    htpasswd -Bbn "$username" "$password" | tr -d '\r'
+    return 0
+  fi
+
+  if command -v docker >/dev/null 2>&1; then
+    docker run --rm --entrypoint htpasswd httpd:2 -Bbn "$username" "$password" | tr -d '\r'
+    return 0
+  fi
+
+  echo "missing command: htpasswd or docker (required to generate registry htpasswd secret)" >&2
+  return 1
+}
+
 load_bootstrap_env() {
   local env_file="${BOOTSTRAP_ENV_FILE:-$BOOTSTRAP_ENV_FILE_DEFAULT}"
   if [[ -f "$env_file" ]]; then
@@ -115,6 +163,52 @@ wait_for_ssh() {
   done
 }
 
+wait_for_kubectl() {
+  local max_attempts="${1:-120}"
+  local sleep_seconds="${2:-5}"
+  local attempt=1
+
+  until kubectl get ns >/dev/null 2>&1; do
+    if (( attempt >= max_attempts )); then
+      echo "timed out waiting for kubectl API access" >&2
+      return 1
+    fi
+    if (( attempt == 1 || attempt % 6 == 0 )); then
+      echo "waiting for kubectl API access (${attempt}/${max_attempts})..."
+    fi
+    sleep "$sleep_seconds"
+    attempt=$((attempt + 1))
+  done
+}
+
+derive_tailscale_control_plane_hostname() {
+  local host_name="$1"
+
+  command -v tailscale >/dev/null 2>&1 || {
+    echo "tailscale CLI is required locally for tailscale-first bootstrap" >&2
+    return 1
+  }
+
+  local tailscale_json
+  tailscale_json="$(mktemp)"
+  tailscale status --json >"$tailscale_json" 2>/dev/null || {
+    rm -f "$tailscale_json"
+    return 1
+  }
+
+  python3 - "$host_name" "$tailscale_json" <<'PY'
+import json,sys
+host=sys.argv[1]
+path=sys.argv[2]
+with open(path) as fh:
+    data=json.load(fh)
+suffix=data.get('MagicDNSSuffix') or ''
+if suffix:
+    print(f"{host}.{suffix}")
+PY
+  rm -f "$tailscale_json"
+}
+
 wait_for_tailscale_node() {
   local host_name="$1"
   local max_attempts="${2:-120}"
@@ -127,12 +221,16 @@ wait_for_tailscale_node() {
   }
 
   while true; do
-    local discovered
+    local discovered tailscale_json
-    discovered=$(tailscale status --json 2>/dev/null | python3 - "$host_name" <<'PY'
+    tailscale_json="$(mktemp)"
+    if tailscale status --json >"$tailscale_json" 2>/dev/null; then
+      discovered=$(python3 - "$host_name" "$tailscale_json" <<'PY'
 import json,sys
 host=sys.argv[1]
+path=sys.argv[2]
 try:
-    data=json.load(sys.stdin)
+    with open(path) as fh:
+        data=json.load(fh)
 except Exception:
     print("")
     raise SystemExit(0)
@@ -141,19 +239,23 @@ matches=[]
 for peer in peers.values():
     if peer.get('HostName') == host:
         matches.append(peer)
-for peer in sorted(matches, key=lambda p: ((p.get('Online') is True), p.get('DNSName') or ''), reverse=True):
+for peer in sorted(matches, key=lambda p: (p.get('DNSName') or ''), reverse=True):
-    if peer.get('Online'):
+    if not peer.get('Online'):
+        continue
     dns=(peer.get('DNSName') or '').rstrip('.')
     if dns:
         print(dns)
         raise SystemExit(0)
-for peer in sorted(matches, key=lambda p: p.get('DNSName') or '', reverse=True):
     if peer.get('TailscaleIPs'):
         print(peer['TailscaleIPs'][0])
         raise SystemExit(0)
 print("")
 PY
 )
+    else
+      discovered=""
+    fi
+    rm -f "$tailscale_json"
     if [[ -n "$discovered" ]]; then
       printf '%s\n' "$discovered"
       return 0

scripts/hetzner/seed-forgejo-repo.sh

@@ -9,20 +9,35 @@ load_bootstrap_env
 resolve_secret_var FORGEJO_ADMIN_PASSWORD required
 
 : "${FORGEJO_ROOT_URL:?set FORGEJO_ROOT_URL}"
+: "${FORGEJO_PUSH_URL_BASE:=$FORGEJO_ROOT_URL}"
 : "${FORGEJO_ADMIN_USERNAME:?set FORGEJO_ADMIN_USERNAME}"
 : "${FORGEJO_REPO_OWNER:=$FORGEJO_ADMIN_USERNAME}"
 : "${FORGEJO_REPO_NAME:=$(basename "$ROOT_DIR")}"
 : "${FORGEJO_PUSH_REMOTE_NAME:=forgejo}"
 : "${FORGEJO_PUSH_REF:=HEAD:refs/heads/main}"
+: "${FORGEJO_REPO_HTTP_USERNAME:=$FORGEJO_ADMIN_USERNAME}"
 
 require git
 
+urlencode() {
+  python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$1"
+}
+
 remote_url="${FORGEJO_ROOT_URL%/}/${FORGEJO_REPO_OWNER}/${FORGEJO_REPO_NAME}.git"
+auth_remote_url="${FORGEJO_PUSH_URL_BASE%/}/${FORGEJO_REPO_OWNER}/${FORGEJO_REPO_NAME}.git"
+encoded_username="$(urlencode "$FORGEJO_REPO_HTTP_USERNAME")"
+if [[ -n "${FORGEJO_ADMIN_PASSWORD:-}" ]]; then
+  encoded_password="$(urlencode "$FORGEJO_ADMIN_PASSWORD")"
+  auth_remote_url="${FORGEJO_PUSH_URL_BASE%/}"
+  auth_remote_url="${auth_remote_url/https:\/\//https://${encoded_username}:${encoded_password}@}"
+  auth_remote_url="${auth_remote_url/http:\/\//http://${encoded_username}:${encoded_password}@}"
+  auth_remote_url+="/${FORGEJO_REPO_OWNER}/${FORGEJO_REPO_NAME}.git"
+fi
 current_remote_url="$(git remote get-url "$FORGEJO_PUSH_REMOTE_NAME" 2>/dev/null || true)"
 if [[ -z "$current_remote_url" ]]; then
-  git remote add "$FORGEJO_PUSH_REMOTE_NAME" "$remote_url"
+  git remote add "$FORGEJO_PUSH_REMOTE_NAME" "$auth_remote_url"
-elif [[ "$current_remote_url" != "$remote_url" ]]; then
+elif [[ "$current_remote_url" != "$auth_remote_url" ]]; then
-  git remote set-url "$FORGEJO_PUSH_REMOTE_NAME" "$remote_url"
+  git remote set-url "$FORGEJO_PUSH_REMOTE_NAME" "$auth_remote_url"
 fi
 
 askpass_script="$(mktemp)"