feat: add headlamp web ui for cluster ops

deploy/k8s/README.md

@@ -12,7 +12,7 @@ This directory is the repo-driven deployment target for the single-node Hetzner+
 Shared platform namespaces:
 - `forgejo`
 - `registry`
-- `observability`
+- `observability` (`grafana`, `loki`, `promtail`, `headlamp`)
 - `ingress-nginx`
 - `cert-manager`
 

deploy/k8s/overlays/hetzner-single-node/ingress-hosts.patch.yaml

@@ -63,3 +63,25 @@ spec:
                 name: grafana
                 port:
                   number: 3000
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: headlamp
+  namespace: observability
+spec:
+  tls:
+    - hosts:
+        - headlamp.doran.133011.xyz
+      secretName: headlamp-tls
+  rules:
+    - host: headlamp.doran.133011.xyz
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: headlamp
+                port:
+                  number: 80

deploy/k8s/platform/base/headlamp.yaml

@@ -0,0 +1,100 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: headlamp-admin
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: headlamp
+    app.kubernetes.io/part-of: observability
+    project.pi.io/type: platform
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: headlamp-admin
+  labels:
+    app.kubernetes.io/name: headlamp
+    app.kubernetes.io/part-of: observability
+    project.pi.io/type: platform
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: headlamp-admin
+    namespace: observability
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: headlamp-admin-token
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: headlamp
+    app.kubernetes.io/part-of: observability
+    project.pi.io/type: platform
+  annotations:
+    kubernetes.io/service-account.name: headlamp-admin
+type: kubernetes.io/service-account-token
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: headlamp
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: headlamp
+    app.kubernetes.io/part-of: observability
+    project.pi.io/type: platform
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: headlamp
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: headlamp
+        app.kubernetes.io/part-of: observability
+    spec:
+      containers:
+        - name: headlamp
+          image: ghcr.io/headlamp-k8s/headlamp:v0.41.0
+          args:
+            - -in-cluster
+            - -plugins-dir=/headlamp/plugins
+          ports:
+            - name: http
+              containerPort: 4466
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+            initialDelaySeconds: 20
+            timeoutSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+            initialDelaySeconds: 30
+            timeoutSeconds: 10
+      nodeSelector:
+        kubernetes.io/os: linux
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: headlamp
+  namespace: observability
+  labels:
+    app.kubernetes.io/name: headlamp
+    app.kubernetes.io/part-of: observability
+    project.pi.io/type: platform
+spec:
+  selector:
+    app.kubernetes.io/name: headlamp
+  ports:
+    - name: http
+      port: 80
+      targetPort: http

deploy/k8s/platform/base/ingress.yaml

@@ -72,3 +72,28 @@ spec:
                 name: grafana
                 port:
                   number: 3000
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: headlamp
+  namespace: observability
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-production
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - headlamp.example.invalid
+      secretName: headlamp-tls
+  rules:
+    - host: headlamp.example.invalid
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: headlamp
+                port:
+                  number: 80

deploy/k8s/platform/base/kustomization.yaml

@@ -4,6 +4,7 @@ resources:
   - namespace.yaml
   - traefik-config.yaml
   - observability.yaml
+  - headlamp.yaml
   - forgejo.yaml
   - forgejo-rbac.yaml
   - forgejo-runner.yaml

deploy/k8s/secrets/README.md

@@ -30,6 +30,8 @@ kubectl apply -k deploy/k8s/overlays/hetzner-single-node
 
 The Forgejo runner no longer expects a pre-seeded `runner_registration_token` secret; `scripts/hetzner/bootstrap.sh` generates a one-time token in-cluster, registers the runner, stores the resulting `/data/.runner` config on the `forgejo-runner-data` PVC, and then restarts the deployment.
 
+Headlamp login is different: its Kubernetes service-account token is generated in-cluster from `deploy/k8s/platform/base/headlamp.yaml` and bootstrap can optionally store that token in `pass` via `HEADLAMP_ADMIN_TOKEN_PASS`. It is not sourced from a checked-in env file.
+
 For future projects, follow the same convention with project-specific secret names in project-specific namespaces.
 
 Do not commit populated secret files.

docs/hetzner-k3s-bootstrap.md

@@ -11,7 +11,7 @@ Goal: provision and deploy everything from this repo to a single Hetzner machine
   - trading system services
   - private registry
   - Forgejo
-  - Loki + Promtail + Grafana observability
+  - Loki + Promtail + Grafana + Headlamp observability
   - k3s-bundled Traefik ingress resources
   - cert-manager
   - ACME issuers
@@ -92,6 +92,7 @@ Required values:
 - `REGISTRY_DOMAIN`
 - `GRAFANA_DOMAIN`
 - `GRAFANA_ROOT_URL`
+- `HEADLAMP_DOMAIN`
 - `LETSENCRYPT_EMAIL`
 - `REGISTRY_USERNAME`
 - `REGISTRY_PASSWORD_PASS` or `REGISTRY_PASSWORD`
@@ -147,6 +148,7 @@ If DNS provider credentials are present, bootstrap updates:
 - `git.${PUBLIC_DOMAIN}`
 - `registry.${PUBLIC_DOMAIN}`
 - `grafana.${PUBLIC_DOMAIN}`
+- `headlamp.${PUBLIC_DOMAIN}`
 
 Supported scripted providers:
 - Cloudflare

docs/hetzner-self-hosted-ci-runbook.md

@@ -23,6 +23,7 @@ After that you should have:
 - the current repo pushed to Forgejo automatically in default mode
 - Registry reachable at `https://${REGISTRY_DOMAIN}`
 - Grafana reachable at `https://${GRAFANA_DOMAIN}`
+- Headlamp reachable at `https://${HEADLAMP_DOMAIN}`
 - private admin/control-plane access over Tailscale if configured
 
 Bootstrap repo automation requires `FORGEJO_ADMIN_USERNAME`, `FORGEJO_ADMIN_PASSWORD`, Python `PyYAML` locally for kubeconfig rendering, and Python `PyNaCl` locally in the default `forgejo-actions` mode so the script can encrypt Forgejo Actions secrets before upload. Bootstrap now fails fast with an explicit preflight error if those Python modules are missing. The same bootstrap flow now also creates the initial Forgejo admin account and writes a durable `/data/.runner` config into the shared Forgejo PVC before the runner deployment is allowed to start.
@@ -39,7 +40,7 @@ kubectl get nodes -o wide
 kubectl get pods -A
 kubectl -n forgejo get deploy,pods,svc,ingress
 kubectl -n registry get deploy,pods,svc,ingress
-kubectl -n observability get deploy,ds,pods,svc,ingress
+kubectl -n observability get deploy,ds,pods,svc,ingress,secrets
 kubectl -n unrip get deploy,pods
 ```
 
@@ -132,7 +133,7 @@ Likewise, generated local kubeconfigs/manifests remain on disk unless you set `D
 
 TLS is issued by cert-manager using the rendered Let's Encrypt email and ingress hosts.
 
-For log inspection in the browser, use Grafana/Loki as documented in `docs/k8s-observability.md`.
+For browser-based cluster inspection and pod logs, use Headlamp. For historical log search, use Grafana/Loki. Both are documented in `docs/k8s-observability.md`.
 
 ## Current limitations
 - the bootstrap path now creates the initial admin account and runner config automatically from inside the Forgejo pod, but it still depends on the operator supplying the intended admin credentials up front

docs/k8s-observability.md

@@ -1,10 +1,11 @@
 # Kubernetes observability on the Hetzner single-node cluster
 
-This cluster now includes a minimal reproducible log stack in the `observability` namespace:
+This cluster now includes a reproducible ops/observability stack in the `observability` namespace:
 
 - `loki` for log storage and querying
 - `promtail` as a DaemonSet that ships pod stdout/stderr logs from every node
-- `grafana` as the web UI
+- `grafana` for log search and historical exploration
+- `headlamp` for a Kubernetes web UI with pods, workloads, events, and pod logs
 
 ## What gets collected
 
@@ -24,7 +25,7 @@ Grafana is exposed through Traefik + cert-manager at:
 - `https://${GRAFANA_DOMAIN}` when bootstrapped from `scripts/hetzner/bootstrap-secrets.env`
 - in the current live environment: `https://grafana.doran.133011.xyz/`
 
-Admin credentials come from:
+Grafana credentials come from:
 
 - `GRAFANA_ADMIN_USERNAME`
 - `GRAFANA_ADMIN_PASSWORD_PASS` or `GRAFANA_ADMIN_PASSWORD`
@@ -34,11 +35,22 @@ In the current live setup the password is stored at:
 
 - `api/hetznerk3s/grafana-admin-password`
 
+Headlamp is exposed at:
+
+- `https://${HEADLAMP_DOMAIN}` when bootstrapped from `scripts/hetzner/bootstrap-secrets.env`
+- in the current live environment: `https://headlamp.doran.133011.xyz/`
+
+Headlamp uses a Kubernetes service-account token for login. Bootstrap stores the generated token in `pass` when `HEADLAMP_ADMIN_TOKEN_PASS` is set.
+In the current live setup it is stored at:
+
+- `api/hetznerk3s/headlamp-admin-token`
+
 ## Reproducible bootstrap path
 
 The observability stack is part of the repo-managed platform layer:
 
 - `deploy/k8s/platform/base/observability.yaml`
+- `deploy/k8s/platform/base/headlamp.yaml`
 - `deploy/k8s/platform/base/kustomization.yaml`
 - `deploy/k8s/platform/base/namespace.yaml`
 - `deploy/k8s/overlays/hetzner-single-node/storage-class.patch.yaml`
@@ -46,11 +58,13 @@ The observability stack is part of the repo-managed platform layer:
 - `deploy/k8s/overlays/hetzner-single-node/ingress-hosts.patch.yaml`
 - `deploy/k8s/overlays/hetzner-single-node/secrets/observability.env.example`
 
-Bootstrap materializes the Grafana secret from local env / `pass`:
+Bootstrap materializes the Grafana secret from local env / `pass` and also stores the generated Headlamp login token back into `pass` when configured:
 
 - writes `deploy/k8s/overlays/hetzner-single-node/secrets/observability.env`
 - copies it into `.state/hetzner/generated-overlay/`
 - applies the generated overlay
+- waits for `headlamp-admin-token`
+- stores that token via `HEADLAMP_ADMIN_TOKEN_PASS`
 
 ## Verify the stack
 
@@ -62,6 +76,7 @@ kubectl -n observability get pvc
 kubectl -n observability get ingress
 kubectl -n observability rollout status deployment/loki --timeout=300s
 kubectl -n observability rollout status deployment/grafana --timeout=300s
+kubectl -n observability rollout status deployment/headlamp --timeout=300s
 kubectl -n observability rollout status daemonset/promtail --timeout=300s
 ```
 
@@ -84,6 +99,20 @@ curl -G -sS 'http://127.0.0.1:3100/loki/api/v1/query' \
 
 If those queries return labels/streams, pod logs are reaching Loki.
 
+## Use Headlamp
+
+1. open `https://headlamp.doran.133011.xyz/`
+2. fetch the login token with:
+
+```bash
+pass show api/hetznerk3s/headlamp-admin-token
+```
+
+3. paste that token into the Headlamp login form
+4. browse namespaces, workloads, pods, and use the built-in pod log view
+
+For this disposable cluster the generated Headlamp token is bound to `cluster-admin` so the UI can show everything. For a production setup, replace that with narrower RBAC.
+
 ## Use Grafana
 
 After logging into Grafana:
@@ -115,11 +144,17 @@ kubectl -n forgejo logs deploy/forgejo -f
 bash scripts/k8s/logs.sh
 ```
 
+Use Headlamp when you want:
+- a web UI listing workloads and pods
+- click-through pod inspection
+- built-in pod log viewing
+- events and resource browsing
+
 Use Grafana when you want:
-- a browser UI
 - historical log search
-- multi-namespace filtering
+- cross-pod filtering
-- easier cross-pod inspection
+- LogQL queries
+- easier multi-namespace log exploration
 
 ## Security notes
 
@@ -128,7 +163,7 @@ For this cluster it is publicly reachable behind Grafana login.
 That is acceptable for this disposable single-node setup, but for a harder production posture prefer one of:
 
 - Tailscale-only access
-- ingress auth in front of Grafana
+- ingress auth in front of Grafana and Headlamp
 - SSO/OIDC
 
 ## Add a new app and have logs show up there

scripts/hetzner/bootstrap-secrets.env.example

@@ -57,6 +57,7 @@ export FORGEJO_ROOT_URL="${FORGEJO_ROOT_URL:-https://${FORGEJO_DOMAIN}/}"
 export REGISTRY_DOMAIN="${REGISTRY_DOMAIN:-registry.${PUBLIC_DOMAIN}}"
 export GRAFANA_DOMAIN="${GRAFANA_DOMAIN:-grafana.${PUBLIC_DOMAIN}}"
 export GRAFANA_ROOT_URL="${GRAFANA_ROOT_URL:-https://${GRAFANA_DOMAIN}/}"
+export HEADLAMP_DOMAIN="${HEADLAMP_DOMAIN:-headlamp.${PUBLIC_DOMAIN}}"
 export LETSENCRYPT_EMAIL="${LETSENCRYPT_EMAIL:-ops@example.com}"
 
 # Optional DNS automation: choose one provider
@@ -84,6 +85,10 @@ export FORGEJO_ADMIN_PASSWORD_PASS="${FORGEJO_ADMIN_PASSWORD_PASS:-$(pass_ref fo
 export GRAFANA_ADMIN_USERNAME="${GRAFANA_ADMIN_USERNAME:-admin}"
 export GRAFANA_ADMIN_PASSWORD_PASS="${GRAFANA_ADMIN_PASSWORD_PASS:-$(pass_ref grafana/admin-password)}"
 
+# Optional storage path for the generated Headlamp admin login token.
+# Bootstrap writes the in-cluster token here after Headlamp is available.
+export HEADLAMP_ADMIN_TOKEN_PASS="${HEADLAMP_ADMIN_TOKEN_PASS:-$(pass_ref headlamp/admin-token)}"
+
 # Optional explicit overrides for CI/testing:
 # export HCLOUD_TOKEN="..."
 # export REGISTRY_PASSWORD="..."

scripts/hetzner/bootstrap.sh

@@ -47,6 +47,7 @@ resolve_secret_var PORKBUN_SECRET_API_KEY optional
 : "${REGISTRY_DOMAIN:=registry.${PUBLIC_DOMAIN}}"
 : "${GRAFANA_DOMAIN:=grafana.${PUBLIC_DOMAIN}}"
 : "${GRAFANA_ROOT_URL:=https://${GRAFANA_DOMAIN}/}"
+: "${HEADLAMP_DOMAIN:=headlamp.${PUBLIC_DOMAIN}}"
 : "${GRAFANA_ADMIN_USERNAME:=admin}"
 : "${REGISTRY_USERNAME:?set REGISTRY_USERNAME}"
 : "${TAILSCALE_CONTROL_PLANE_HOSTNAME:=}"
@@ -332,6 +333,28 @@ spec:
                 name: grafana
                 port:
                   number: 3000
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: headlamp
+  namespace: observability
+spec:
+  tls:
+    - hosts:
+        - {"$HEADLAMP_DOMAIN"}
+      secretName: headlamp-tls
+  rules:
+    - host: {"$HEADLAMP_DOMAIN"}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: headlamp
+                port:
+                  number: 80
 ''')
 PY
 
@@ -356,9 +379,28 @@ kubectl -n forgejo rollout status deployment/forgejo --timeout=300s
 kubectl -n registry rollout status deployment/registry --timeout=300s
 kubectl -n observability rollout status deployment/loki --timeout=300s
 kubectl -n observability rollout status deployment/grafana --timeout=300s
+kubectl -n observability rollout status deployment/headlamp --timeout=300s
 kubectl -n observability rollout status daemonset/promtail --timeout=300s
 kubectl -n "$PROJECT_NAMESPACE" rollout status deployment/redpanda --timeout=300s
 
+HEADLAMP_ADMIN_TOKEN=""
+for attempt in $(seq 1 60); do
+  HEADLAMP_ADMIN_TOKEN="$(kubectl -n observability get secret headlamp-admin-token -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null || true)"
+  if [[ -n "$HEADLAMP_ADMIN_TOKEN" ]]; then
+    break
+  fi
+  if (( attempt == 1 || attempt % 6 == 0 )); then
+    echo "waiting for headlamp admin token (${attempt}/60)..."
+  fi
+  sleep 2
+done
+if [[ -z "$HEADLAMP_ADMIN_TOKEN" ]]; then
+  echo "warning: headlamp admin token not available yet; rerun bootstrap or read secret headlamp-admin-token manually" >&2
+elif [[ -n "${HEADLAMP_ADMIN_TOKEN_PASS:-}" ]]; then
+  store_secret_to_pass "$HEADLAMP_ADMIN_TOKEN_PASS" "$HEADLAMP_ADMIN_TOKEN"
+  echo "stored headlamp admin token in pass: $HEADLAMP_ADMIN_TOKEN_PASS"
+fi
+
 forgejo_admin_user_b64=$(printf '%s' "$FORGEJO_ADMIN_USERNAME" | base64 | tr -d '\n')
 forgejo_admin_pass_b64=$(printf '%s' "$FORGEJO_ADMIN_PASSWORD" | base64 | tr -d '\n')
 forgejo_admin_email_b64=$(printf '%s' "$FORGEJO_ADMIN_EMAIL" | base64 | tr -d '\n')
@@ -513,4 +555,6 @@ echo "forgejo_url=$FORGEJO_ROOT_URL"
 echo "forgejo_repo=${FORGEJO_ROOT_URL%/}/$FORGEJO_REPO_OWNER/$FORGEJO_REPO_NAME"
 echo "registry_url=https://$REGISTRY_DOMAIN"
 echo "grafana_url=$GRAFANA_ROOT_URL"
+echo "headlamp_url=https://$HEADLAMP_DOMAIN/"
+echo "headlamp_token_pass=${HEADLAMP_ADMIN_TOKEN_PASS:-}"
 echo "dns_provider=${CLOUDFLARE_API_TOKEN:+cloudflare}${PORKBUN_API_KEY:+porkbun}"

scripts/hetzner/configure-cloudflare-dns.sh

@@ -59,6 +59,7 @@ records=(
   "git.$PUBLIC_DOMAIN"
   "registry.$PUBLIC_DOMAIN"
   "grafana.$PUBLIC_DOMAIN"
+  "headlamp.$PUBLIC_DOMAIN"
 )
 
 case "$DNS_MODE" in
@@ -68,6 +69,7 @@ case "$DNS_MODE" in
     upsert_record A "${records[1]}" "$SERVER_IP" false
     upsert_record A "${records[2]}" "$SERVER_IP" false
     upsert_record A "${records[3]}" "$SERVER_IP" false
+    upsert_record A "${records[4]}" "$SERVER_IP" false
     echo "cloudflare dns updated for ${records[*]}"
     ;;
   delete)
@@ -75,6 +77,7 @@ case "$DNS_MODE" in
     delete_record A "${records[1]}"
     delete_record A "${records[2]}"
     delete_record A "${records[3]}"
+    delete_record A "${records[4]}"
     echo "cloudflare dns cleanup finished for ${records[*]}"
     ;;
   *)

scripts/hetzner/configure-porkbun-dns.sh

@@ -28,10 +28,12 @@ if [[ -n "$root_name" ]]; then
   git_name="git.$root_name"
   registry_name="registry.$root_name"
   grafana_name="grafana.$root_name"
+  headlamp_name="headlamp.$root_name"
 else
   git_name="git"
   registry_name="registry"
   grafana_name="grafana"
+  headlamp_name="headlamp"
 fi
 
 payload() {
@@ -117,14 +119,16 @@ case "$DNS_MODE" in
     upsert_a_record "$git_name"
     upsert_a_record "$registry_name"
     upsert_a_record "$grafana_name"
-    echo "porkbun dns updated for $PUBLIC_DOMAIN, git.$PUBLIC_DOMAIN, registry.$PUBLIC_DOMAIN, grafana.$PUBLIC_DOMAIN"
+    upsert_a_record "$headlamp_name"
+    echo "porkbun dns updated for $PUBLIC_DOMAIN, git.$PUBLIC_DOMAIN, registry.$PUBLIC_DOMAIN, grafana.$PUBLIC_DOMAIN, headlamp.$PUBLIC_DOMAIN"
     ;;
   delete)
     delete_a_record "$root_name"
     delete_a_record "$git_name"
     delete_a_record "$registry_name"
     delete_a_record "$grafana_name"
-    echo "porkbun dns cleanup finished for $PUBLIC_DOMAIN, git.$PUBLIC_DOMAIN, registry.$PUBLIC_DOMAIN, grafana.$PUBLIC_DOMAIN"
+    delete_a_record "$headlamp_name"
+    echo "porkbun dns cleanup finished for $PUBLIC_DOMAIN, git.$PUBLIC_DOMAIN, registry.$PUBLIC_DOMAIN, grafana.$PUBLIC_DOMAIN, headlamp.$PUBLIC_DOMAIN"
     ;;
   *)
     echo "unsupported DNS_MODE: $DNS_MODE" >&2