From b422c98b53f3929b4da2ca307062396b5f8729c9 Mon Sep 17 00:00:00 2001
From: Philipp <philipp@doran.133011.xyz>
Date: Mon, 30 Mar 2026 17:57:49 +0200
Subject: [PATCH] fix: bootstrap standalone app repo on cluster rebuild

---
 deploy/k8s/platform/base/observability.yaml   |  8 +-
 scripts/hetzner/bootstrap-secrets.env.example |  9 ++-
 scripts/hetzner/bootstrap.sh                  | 80 +++++++++++++++++--
 scripts/hetzner/seed-forgejo-repo.sh          | 18 +++--
 4 files changed, 97 insertions(+), 18 deletions(-)

diff --git a/deploy/k8s/platform/base/observability.yaml b/deploy/k8s/platform/base/observability.yaml
index 07a1a63..a2d97d7 100644
--- a/deploy/k8s/platform/base/observability.yaml
+++ b/deploy/k8s/platform/base/observability.yaml
@@ -42,8 +42,8 @@ data:
     limits_config:
       allow_structured_metadata: false
       reject_old_samples: true
-      reject_old_samples_max_age: 168h
-      retention_period: 168h
+      reject_old_samples_max_age: 48h
+      retention_period: 48h
 
     compactor:
       working_directory: /var/loki/compactor
@@ -225,6 +225,10 @@ data:
         pipeline_stages:
           - cri: {}
         relabel_configs:
+          - source_labels:
+              - __meta_kubernetes_namespace
+            regex: kube-system|observability
+            action: drop
           - source_labels:
               - __meta_kubernetes_pod_node_name
             target_label: __host__
diff --git a/scripts/hetzner/bootstrap-secrets.env.example b/scripts/hetzner/bootstrap-secrets.env.example
index 410f596..72b4163 100644
--- a/scripts/hetzner/bootstrap-secrets.env.example
+++ b/scripts/hetzner/bootstrap-secrets.env.example
@@ -30,17 +30,20 @@ pass_ref() {
 export HCLOUD_TOKEN_PASS="${HCLOUD_TOKEN_PASS:-$(pass_ref hetzner/hcloud-token)}"
 export SSH_PUBLIC_KEY_PATH="${SSH_PUBLIC_KEY_PATH:-$HOME/.ssh/id_ed25519.pub}"
 
-# Optional project defaults. The infra repo still prepares the shared unrip namespace,
-# secrets, and registry auth by default, but the app manifests now live in the separate
-# unrip repository.
+# Optional project defaults. The infra repo prepares the shared unrip namespace,
+# secrets, and registry auth. The app code/manifests are expected in a separate repo.
 export PROJECT_NAME="${PROJECT_NAME:-unrip}"
 export PROJECT_NAMESPACE="${PROJECT_NAMESPACE:-$PROJECT_NAME}"
+export APP_REPO_DIR="${APP_REPO_DIR:-$PWD/../unrip-project}"
 # export PROJECT_OVERLAY_DIR="$PWD/deploy/k8s/overlays/hetzner-single-node"
 # export PROJECT_SECRET_NAME="unrip-secrets"
 # export PROJECT_SECRET_ENV_BASENAME="unrip.env"
 # export PROJECT_REGISTRY_SECRET_NAME="unrip-registry-creds"
 # export PROJECT_IMAGE_REPOSITORY="unrip"
 # export PROJECT_DEPLOYMENTS="near-intents-ingest dummy-reactor dummy-executor dummy-consumer"
+# export APP_REPO_KUSTOMIZE_PATH="deploy/k8s/base"
+# export APP_FORGEJO_REPO_OWNER="$FORGEJO_ADMIN_USERNAME"
+# export APP_FORGEJO_REPO_NAME="$PROJECT_NAME"
 
 # Tailscale-first admin access (recommended)
 export TAILSCALE_AUTH_KEY_PASS="${TAILSCALE_AUTH_KEY_PASS:-$(pass_ref tailscale/auth-key)}"
diff --git a/scripts/hetzner/bootstrap.sh b/scripts/hetzner/bootstrap.sh
index 02aa876..af4e4c8 100755
--- a/scripts/hetzner/bootstrap.sh
+++ b/scripts/hetzner/bootstrap.sh
@@ -73,15 +73,30 @@ resolve_secret_var PORKBUN_SECRET_API_KEY optional
 : "${FORGEJO_REPO_OWNER:=$FORGEJO_ADMIN_USERNAME}"
 : "${FORGEJO_REPO_NAME:=$(basename "$ROOT_DIR")}"
 : "${FORGEJO_REPO_PRIVATE:=true}"
+: "${APP_REPO_DIR:=$(realpath "$ROOT_DIR/../unrip-project")}"
+: "${APP_REPO_KUSTOMIZE_PATH:=deploy/k8s/base}"
+: "${APP_FORGEJO_REPO_OWNER:=$FORGEJO_REPO_OWNER}"
+: "${APP_FORGEJO_REPO_NAME:=$PROJECT_NAME}"
+: "${APP_FORGEJO_REPO_PRIVATE:=true}"
 : "${BOOTSTRAP_DELIVERY_MODE:=forgejo-actions}"
 
 BOOTSTRAP_IMAGE="${PROJECT_IMAGE_REPOSITORY}:bootstrap"
 PROJECT_SECRET_ENV_PATH="$PROJECT_OVERLAY_DIR/secrets/$PROJECT_SECRET_ENV_BASENAME"
 GENERATED_OVERLAY_DIR="$STATE_DIR/generated-overlay"
+APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
+APP_KUSTOMIZE_DIR="$APP_REPO_DIR/$APP_REPO_KUSTOMIZE_PATH"
 
 if [[ "$BOOTSTRAP_DELIVERY_MODE" != "forgejo-actions" ]]; then
   require docker
 fi
+if [[ ! -d "$APP_REPO_DIR/.git" ]]; then
+  echo "missing app repository at $APP_REPO_DIR" >&2
+  exit 1
+fi
+if [[ ! -f "$APP_KUSTOMIZE_DIR/kustomization.yaml" ]]; then
+  echo "missing app kustomization at $APP_KUSTOMIZE_DIR/kustomization.yaml" >&2
+  exit 1
+fi
 if [[ -n "${TAILSCALE_AUTH_KEY:-}" && "$TF_ADMIN_CIDR_BLOCKS" == '[]' && "$BOOTSTRAP_ALLOW_PUBLIC_ADMIN_FALLBACK" == "1" ]]; then
   OPERATOR_PUBLIC_IP="$(curl -fsS https://api.ipify.org || true)"
   if [[ "$OPERATOR_PUBLIC_IP" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
@@ -383,7 +398,10 @@ kubectl -n observability rollout status deployment/loki --timeout=300s
 kubectl -n observability rollout status deployment/grafana --timeout=300s
 kubectl -n observability rollout status deployment/headlamp --timeout=300s
 kubectl -n observability rollout status daemonset/promtail --timeout=300s
+
+kubectl apply -k "$APP_KUSTOMIZE_DIR"
 kubectl -n "$PROJECT_NAMESPACE" rollout status deployment/redpanda --timeout=300s
+kubectl -n "$PROJECT_NAMESPACE" wait --for=condition=Complete --timeout=300s job/redpanda-topic-bootstrap
 
 HEADLAMP_ADMIN_TOKEN=""
 for attempt in $(seq 1 60); do
@@ -481,7 +499,8 @@ wait_for_url "$FORGEJO_BOOTSTRAP_URL" "Forgejo bootstrap URL" 60 2
 
 if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then
   FORGEJO_ADMIN_API_TOKEN="$(kubectl -n forgejo exec deploy/forgejo -- /bin/bash --noprofile --norc -lc "su-exec git /usr/local/bin/forgejo admin user generate-access-token --config /data/gitea/conf/app.ini --username '$FORGEJO_ADMIN_USERNAME' --token-name bootstrap-$(date +%s) --scopes read:user,read:repository,write:repository,write:user --raw" | tr -d '\r\n')"
-  forgejo_bootstrap_args=(
+
+  infra_bootstrap_args=(
     --forgejo-url "$FORGEJO_BOOTSTRAP_URL"
     --token "$FORGEJO_ADMIN_API_TOKEN"
     --admin-username "$FORGEJO_ADMIN_USERNAME"
@@ -497,27 +516,73 @@ if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then
     --project-path "$PROJECT_REPO_PATH"
   )
   if [[ "$FORGEJO_REPO_PRIVATE" == "true" ]]; then
-    forgejo_bootstrap_args+=(--repo-private)
+    infra_bootstrap_args+=(--repo-private)
   fi
-  python3 "$ROOT_DIR/scripts/hetzner/forgejo-bootstrap.py" "${forgejo_bootstrap_args[@]}"
+  python3 "$ROOT_DIR/scripts/hetzner/forgejo-bootstrap.py" "${infra_bootstrap_args[@]}"
 
   FORGEJO_PUSH_URL_BASE="$FORGEJO_BOOTSTRAP_URL" bash "$ROOT_DIR/scripts/hetzner/seed-forgejo-repo.sh"
 
+  app_bootstrap_args=(
+    --forgejo-url "$FORGEJO_BOOTSTRAP_URL"
+    --token "$FORGEJO_ADMIN_API_TOKEN"
+    --admin-username "$FORGEJO_ADMIN_USERNAME"
+    --repo-owner "$APP_FORGEJO_REPO_OWNER"
+    --repo-name "$APP_FORGEJO_REPO_NAME"
+    --kubeconfig "$KUBECONFIG_PATH"
+    --registry-username "$REGISTRY_USERNAME"
+    --registry-password "$REGISTRY_PASSWORD"
+    --registry-host "$REGISTRY_DOMAIN"
+    --project-name "$PROJECT_NAME"
+    --project-namespace "$PROJECT_NAMESPACE"
+    --project-deployments "${PROJECT_DEPLOYMENTS// /,}"
+    --project-path .
+  )
+  if [[ "$APP_FORGEJO_REPO_PRIVATE" == "true" ]]; then
+    app_bootstrap_args+=(--repo-private)
+  fi
+  python3 "$ROOT_DIR/scripts/hetzner/forgejo-bootstrap.py" "${app_bootstrap_args[@]}"
+
   wait_for_url "$FORGEJO_ROOT_URL" "Forgejo public URL" 180 5
   wait_for_http_status "https://$REGISTRY_DOMAIN/v2/" "registry public URL" '200|401' 180 5
+
+  APP_COMMIT_SHA="$(git -C "$APP_REPO_DIR" rev-parse HEAD)"
+  APP_BUILD_JOB="image-build-${APP_COMMIT_SHA:0:12}"
+
+  FORGEJO_PUSH_URL_BASE="$FORGEJO_BOOTSTRAP_URL" \
+  SOURCE_REPO_DIR="$APP_REPO_DIR" \
+  FORGEJO_REPO_OWNER="$APP_FORGEJO_REPO_OWNER" \
+  FORGEJO_REPO_NAME="$APP_FORGEJO_REPO_NAME" \
+    bash "$ROOT_DIR/scripts/hetzner/seed-forgejo-repo.sh"
+
+  for attempt in $(seq 1 120); do
+    if kubectl -n "$PROJECT_NAMESPACE" get job "$APP_BUILD_JOB" >/dev/null 2>&1; then
+      break
+    fi
+    if (( attempt == 1 || attempt % 6 == 0 )); then
+      echo "waiting for app build job $APP_BUILD_JOB (${attempt}/120)..."
+    fi
+    sleep 5
+  done
+  if ! kubectl -n "$PROJECT_NAMESPACE" get job "$APP_BUILD_JOB" >/dev/null 2>&1; then
+    echo "app build job did not appear: $APP_BUILD_JOB" >&2
+    exit 1
+  fi
+  kubectl -n "$PROJECT_NAMESPACE" wait --for=condition=Complete --timeout=1200s "job/$APP_BUILD_JOB"
+  kubectl -n "$PROJECT_NAMESPACE" logs "job/$APP_BUILD_JOB"
 else
-  docker build -t "$BOOTSTRAP_IMAGE" "$PROJECT_DIR"
+  docker build -t "$BOOTSTRAP_IMAGE" "$APP_REPO_DIR"
   docker save "$BOOTSTRAP_IMAGE" \
     | ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$SSH_TARGET" 'sudo k3s ctr images import -'
 
   for deployment in $PROJECT_DEPLOYMENTS; do
     kubectl -n "$PROJECT_NAMESPACE" set image "deployment/${deployment}" app="$BOOTSTRAP_IMAGE"
   done
-  for deployment in $PROJECT_DEPLOYMENTS; do
-    kubectl -n "$PROJECT_NAMESPACE" rollout status "deployment/${deployment}" --timeout=180s
-  done
 fi
 
+for deployment in $PROJECT_DEPLOYMENTS; do
+  kubectl -n "$PROJECT_NAMESPACE" rollout status "deployment/${deployment}" --timeout=300s
+done
+
 DURABLE_K3S_API_URL="$K3S_API_URL"
 DURABLE_INSECURE_SKIP_TLS_VERIFY=0
 if [[ "$USE_SSH_TUNNEL_FOR_K3S" == "1" ]]; then
@@ -556,6 +621,7 @@ echo "ci_kubeconfig=$CI_KUBECONFIG_PATH"
 echo "bootstrap_delivery_mode=$BOOTSTRAP_DELIVERY_MODE"
 echo "forgejo_url=$FORGEJO_ROOT_URL"
 echo "forgejo_repo=${FORGEJO_ROOT_URL%/}/$FORGEJO_REPO_OWNER/$FORGEJO_REPO_NAME"
+echo "app_repo=${FORGEJO_ROOT_URL%/}/$APP_FORGEJO_REPO_OWNER/$APP_FORGEJO_REPO_NAME"
 echo "registry_url=https://$REGISTRY_DOMAIN"
 echo "grafana_url=$GRAFANA_ROOT_URL"
 echo "headlamp_url=https://$HEADLAMP_DOMAIN/"
diff --git a/scripts/hetzner/seed-forgejo-repo.sh b/scripts/hetzner/seed-forgejo-repo.sh
index 0373a7b..bc304ec 100755
--- a/scripts/hetzner/seed-forgejo-repo.sh
+++ b/scripts/hetzner/seed-forgejo-repo.sh
@@ -11,14 +11,20 @@ resolve_secret_var FORGEJO_ADMIN_PASSWORD required
 : "${FORGEJO_ROOT_URL:?set FORGEJO_ROOT_URL}"
 : "${FORGEJO_PUSH_URL_BASE:=$FORGEJO_ROOT_URL}"
 : "${FORGEJO_ADMIN_USERNAME:?set FORGEJO_ADMIN_USERNAME}"
+: "${SOURCE_REPO_DIR:=$ROOT_DIR}"
 : "${FORGEJO_REPO_OWNER:=$FORGEJO_ADMIN_USERNAME}"
-: "${FORGEJO_REPO_NAME:=$(basename "$ROOT_DIR")}"
+: "${FORGEJO_REPO_NAME:=$(basename "$SOURCE_REPO_DIR")}"
 : "${FORGEJO_PUSH_REMOTE_NAME:=forgejo}"
 : "${FORGEJO_PUSH_REF:=HEAD:refs/heads/main}"
 : "${FORGEJO_REPO_HTTP_USERNAME:=$FORGEJO_ADMIN_USERNAME}"
 
 require git
 
+if [[ ! -d "$SOURCE_REPO_DIR/.git" ]]; then
+  echo "SOURCE_REPO_DIR is not a git repository: $SOURCE_REPO_DIR" >&2
+  exit 1
+fi
+
 urlencode() {
   python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$1"
 }
@@ -33,11 +39,11 @@ if [[ -n "${FORGEJO_ADMIN_PASSWORD:-}" ]]; then
   auth_remote_url="${auth_remote_url/http:\/\//http://${encoded_username}:${encoded_password}@}"
   auth_remote_url+="/${FORGEJO_REPO_OWNER}/${FORGEJO_REPO_NAME}.git"
 fi
-current_remote_url="$(git remote get-url "$FORGEJO_PUSH_REMOTE_NAME" 2>/dev/null || true)"
+current_remote_url="$(git -C "$SOURCE_REPO_DIR" remote get-url "$FORGEJO_PUSH_REMOTE_NAME" 2>/dev/null || true)"
 if [[ -z "$current_remote_url" ]]; then
-  git remote add "$FORGEJO_PUSH_REMOTE_NAME" "$auth_remote_url"
+  git -C "$SOURCE_REPO_DIR" remote add "$FORGEJO_PUSH_REMOTE_NAME" "$auth_remote_url"
 elif [[ "$current_remote_url" != "$auth_remote_url" ]]; then
-  git remote set-url "$FORGEJO_PUSH_REMOTE_NAME" "$auth_remote_url"
+  git -C "$SOURCE_REPO_DIR" remote set-url "$FORGEJO_PUSH_REMOTE_NAME" "$auth_remote_url"
 fi
 
 askpass_script="$(mktemp)"
@@ -56,6 +62,6 @@ GIT_TERMINAL_PROMPT=0 \
 GIT_ASKPASS="$askpass_script" \
 FORGEJO_ADMIN_USERNAME="$FORGEJO_ADMIN_USERNAME" \
 FORGEJO_ADMIN_PASSWORD="$FORGEJO_ADMIN_PASSWORD" \
-  git push "$FORGEJO_PUSH_REMOTE_NAME" "$FORGEJO_PUSH_REF"
+  git -C "$SOURCE_REPO_DIR" push "$FORGEJO_PUSH_REMOTE_NAME" "$FORGEJO_PUSH_REF"
 
-echo "seeded ${remote_url}"
+echo "seeded ${remote_url} from ${SOURCE_REPO_DIR}"