fix: bootstrap standalone app repo on cluster rebuild
This commit is contained in:
Philipp
parent
d959725c37
commit
b422c98b53
4 changed files with 97 additions and 18 deletions
|
|
@ -42,8 +42,8 @@ data:
|
|||
limits_config:
|
||||
allow_structured_metadata: false
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
retention_period: 168h
|
||||
reject_old_samples_max_age: 48h
|
||||
retention_period: 48h
|
||||
|
||||
compactor:
|
||||
working_directory: /var/loki/compactor
|
||||
|
|
@ -225,6 +225,10 @@ data:
|
|||
pipeline_stages:
|
||||
- cri: {}
|
||||
relabel_configs:
|
||||
- source_labels:
|
||||
- __meta_kubernetes_namespace
|
||||
regex: kube-system|observability
|
||||
action: drop
|
||||
- source_labels:
|
||||
- __meta_kubernetes_pod_node_name
|
||||
target_label: __host__
|
||||
|
|
|
|||
|
|
@ -30,17 +30,20 @@ pass_ref() {
|
|||
export HCLOUD_TOKEN_PASS="${HCLOUD_TOKEN_PASS:-$(pass_ref hetzner/hcloud-token)}"
|
||||
export SSH_PUBLIC_KEY_PATH="${SSH_PUBLIC_KEY_PATH:-$HOME/.ssh/id_ed25519.pub}"
|
||||
|
||||
# Optional project defaults. The infra repo still prepares the shared unrip namespace,
|
||||
# secrets, and registry auth by default, but the app manifests now live in the separate
|
||||
# unrip repository.
|
||||
# Optional project defaults. The infra repo prepares the shared unrip namespace,
|
||||
# secrets, and registry auth. The app code/manifests are expected in a separate repo.
|
||||
export PROJECT_NAME="${PROJECT_NAME:-unrip}"
|
||||
export PROJECT_NAMESPACE="${PROJECT_NAMESPACE:-$PROJECT_NAME}"
|
||||
export APP_REPO_DIR="${APP_REPO_DIR:-$PWD/../unrip-project}"
|
||||
# export PROJECT_OVERLAY_DIR="$PWD/deploy/k8s/overlays/hetzner-single-node"
|
||||
# export PROJECT_SECRET_NAME="unrip-secrets"
|
||||
# export PROJECT_SECRET_ENV_BASENAME="unrip.env"
|
||||
# export PROJECT_REGISTRY_SECRET_NAME="unrip-registry-creds"
|
||||
# export PROJECT_IMAGE_REPOSITORY="unrip"
|
||||
# export PROJECT_DEPLOYMENTS="near-intents-ingest dummy-reactor dummy-executor dummy-consumer"
|
||||
# export APP_REPO_KUSTOMIZE_PATH="deploy/k8s/base"
|
||||
# export APP_FORGEJO_REPO_OWNER="$FORGEJO_ADMIN_USERNAME"
|
||||
# export APP_FORGEJO_REPO_NAME="$PROJECT_NAME"
|
||||
|
||||
# Tailscale-first admin access (recommended)
|
||||
export TAILSCALE_AUTH_KEY_PASS="${TAILSCALE_AUTH_KEY_PASS:-$(pass_ref tailscale/auth-key)}"
|
||||
|
|
|
|||
|
|
@ -73,15 +73,30 @@ resolve_secret_var PORKBUN_SECRET_API_KEY optional
|
|||
: "${FORGEJO_REPO_OWNER:=$FORGEJO_ADMIN_USERNAME}"
|
||||
: "${FORGEJO_REPO_NAME:=$(basename "$ROOT_DIR")}"
|
||||
: "${FORGEJO_REPO_PRIVATE:=true}"
|
||||
: "${APP_REPO_DIR:=$(realpath "$ROOT_DIR/../unrip-project")}"
|
||||
: "${APP_REPO_KUSTOMIZE_PATH:=deploy/k8s/base}"
|
||||
: "${APP_FORGEJO_REPO_OWNER:=$FORGEJO_REPO_OWNER}"
|
||||
: "${APP_FORGEJO_REPO_NAME:=$PROJECT_NAME}"
|
||||
: "${APP_FORGEJO_REPO_PRIVATE:=true}"
|
||||
: "${BOOTSTRAP_DELIVERY_MODE:=forgejo-actions}"
|
||||
|
||||
BOOTSTRAP_IMAGE="${PROJECT_IMAGE_REPOSITORY}:bootstrap"
|
||||
PROJECT_SECRET_ENV_PATH="$PROJECT_OVERLAY_DIR/secrets/$PROJECT_SECRET_ENV_BASENAME"
|
||||
GENERATED_OVERLAY_DIR="$STATE_DIR/generated-overlay"
|
||||
APP_REPO_DIR="$(realpath "$APP_REPO_DIR")"
|
||||
APP_KUSTOMIZE_DIR="$APP_REPO_DIR/$APP_REPO_KUSTOMIZE_PATH"
|
||||
|
||||
if [[ "$BOOTSTRAP_DELIVERY_MODE" != "forgejo-actions" ]]; then
|
||||
require docker
|
||||
fi
|
||||
if [[ ! -d "$APP_REPO_DIR/.git" ]]; then
|
||||
echo "missing app repository at $APP_REPO_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -f "$APP_KUSTOMIZE_DIR/kustomization.yaml" ]]; then
|
||||
echo "missing app kustomization at $APP_KUSTOMIZE_DIR/kustomization.yaml" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "${TAILSCALE_AUTH_KEY:-}" && "$TF_ADMIN_CIDR_BLOCKS" == '[]' && "$BOOTSTRAP_ALLOW_PUBLIC_ADMIN_FALLBACK" == "1" ]]; then
|
||||
OPERATOR_PUBLIC_IP="$(curl -fsS https://api.ipify.org || true)"
|
||||
if [[ "$OPERATOR_PUBLIC_IP" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
|
|
@ -383,7 +398,10 @@ kubectl -n observability rollout status deployment/loki --timeout=300s
|
|||
kubectl -n observability rollout status deployment/grafana --timeout=300s
|
||||
kubectl -n observability rollout status deployment/headlamp --timeout=300s
|
||||
kubectl -n observability rollout status daemonset/promtail --timeout=300s
|
||||
|
||||
kubectl apply -k "$APP_KUSTOMIZE_DIR"
|
||||
kubectl -n "$PROJECT_NAMESPACE" rollout status deployment/redpanda --timeout=300s
|
||||
kubectl -n "$PROJECT_NAMESPACE" wait --for=condition=Complete --timeout=300s job/redpanda-topic-bootstrap
|
||||
|
||||
HEADLAMP_ADMIN_TOKEN=""
|
||||
for attempt in $(seq 1 60); do
|
||||
|
|
@ -481,7 +499,8 @@ wait_for_url "$FORGEJO_BOOTSTRAP_URL" "Forgejo bootstrap URL" 60 2
|
|||
|
||||
if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then
|
||||
FORGEJO_ADMIN_API_TOKEN="$(kubectl -n forgejo exec deploy/forgejo -- /bin/bash --noprofile --norc -lc "su-exec git /usr/local/bin/forgejo admin user generate-access-token --config /data/gitea/conf/app.ini --username '$FORGEJO_ADMIN_USERNAME' --token-name bootstrap-$(date +%s) --scopes read:user,read:repository,write:repository,write:user --raw" | tr -d '\r\n')"
|
||||
forgejo_bootstrap_args=(
|
||||
|
||||
infra_bootstrap_args=(
|
||||
--forgejo-url "$FORGEJO_BOOTSTRAP_URL"
|
||||
--token "$FORGEJO_ADMIN_API_TOKEN"
|
||||
--admin-username "$FORGEJO_ADMIN_USERNAME"
|
||||
|
|
@ -497,27 +516,73 @@ if [[ "$BOOTSTRAP_DELIVERY_MODE" == "forgejo-actions" ]]; then
|
|||
--project-path "$PROJECT_REPO_PATH"
|
||||
)
|
||||
if [[ "$FORGEJO_REPO_PRIVATE" == "true" ]]; then
|
||||
forgejo_bootstrap_args+=(--repo-private)
|
||||
infra_bootstrap_args+=(--repo-private)
|
||||
fi
|
||||
python3 "$ROOT_DIR/scripts/hetzner/forgejo-bootstrap.py" "${forgejo_bootstrap_args[@]}"
|
||||
python3 "$ROOT_DIR/scripts/hetzner/forgejo-bootstrap.py" "${infra_bootstrap_args[@]}"
|
||||
|
||||
FORGEJO_PUSH_URL_BASE="$FORGEJO_BOOTSTRAP_URL" bash "$ROOT_DIR/scripts/hetzner/seed-forgejo-repo.sh"
|
||||
|
||||
app_bootstrap_args=(
|
||||
--forgejo-url "$FORGEJO_BOOTSTRAP_URL"
|
||||
--token "$FORGEJO_ADMIN_API_TOKEN"
|
||||
--admin-username "$FORGEJO_ADMIN_USERNAME"
|
||||
--repo-owner "$APP_FORGEJO_REPO_OWNER"
|
||||
--repo-name "$APP_FORGEJO_REPO_NAME"
|
||||
--kubeconfig "$KUBECONFIG_PATH"
|
||||
--registry-username "$REGISTRY_USERNAME"
|
||||
--registry-password "$REGISTRY_PASSWORD"
|
||||
--registry-host "$REGISTRY_DOMAIN"
|
||||
--project-name "$PROJECT_NAME"
|
||||
--project-namespace "$PROJECT_NAMESPACE"
|
||||
--project-deployments "${PROJECT_DEPLOYMENTS// /,}"
|
||||
--project-path .
|
||||
)
|
||||
if [[ "$APP_FORGEJO_REPO_PRIVATE" == "true" ]]; then
|
||||
app_bootstrap_args+=(--repo-private)
|
||||
fi
|
||||
python3 "$ROOT_DIR/scripts/hetzner/forgejo-bootstrap.py" "${app_bootstrap_args[@]}"
|
||||
|
||||
wait_for_url "$FORGEJO_ROOT_URL" "Forgejo public URL" 180 5
|
||||
wait_for_http_status "https://$REGISTRY_DOMAIN/v2/" "registry public URL" '200|401' 180 5
|
||||
|
||||
APP_COMMIT_SHA="$(git -C "$APP_REPO_DIR" rev-parse HEAD)"
|
||||
APP_BUILD_JOB="image-build-${APP_COMMIT_SHA:0:12}"
|
||||
|
||||
FORGEJO_PUSH_URL_BASE="$FORGEJO_BOOTSTRAP_URL" \
|
||||
SOURCE_REPO_DIR="$APP_REPO_DIR" \
|
||||
FORGEJO_REPO_OWNER="$APP_FORGEJO_REPO_OWNER" \
|
||||
FORGEJO_REPO_NAME="$APP_FORGEJO_REPO_NAME" \
|
||||
bash "$ROOT_DIR/scripts/hetzner/seed-forgejo-repo.sh"
|
||||
|
||||
for attempt in $(seq 1 120); do
|
||||
if kubectl -n "$PROJECT_NAMESPACE" get job "$APP_BUILD_JOB" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
if (( attempt == 1 || attempt % 6 == 0 )); then
|
||||
echo "waiting for app build job $APP_BUILD_JOB (${attempt}/120)..."
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
if ! kubectl -n "$PROJECT_NAMESPACE" get job "$APP_BUILD_JOB" >/dev/null 2>&1; then
|
||||
echo "app build job did not appear: $APP_BUILD_JOB" >&2
|
||||
exit 1
|
||||
fi
|
||||
kubectl -n "$PROJECT_NAMESPACE" wait --for=condition=Complete --timeout=1200s "job/$APP_BUILD_JOB"
|
||||
kubectl -n "$PROJECT_NAMESPACE" logs "job/$APP_BUILD_JOB"
|
||||
else
|
||||
docker build -t "$BOOTSTRAP_IMAGE" "$PROJECT_DIR"
|
||||
docker build -t "$BOOTSTRAP_IMAGE" "$APP_REPO_DIR"
|
||||
docker save "$BOOTSTRAP_IMAGE" \
|
||||
| ssh -i "$SSH_PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$SSH_TARGET" 'sudo k3s ctr images import -'
|
||||
|
||||
for deployment in $PROJECT_DEPLOYMENTS; do
|
||||
kubectl -n "$PROJECT_NAMESPACE" set image "deployment/${deployment}" app="$BOOTSTRAP_IMAGE"
|
||||
done
|
||||
for deployment in $PROJECT_DEPLOYMENTS; do
|
||||
kubectl -n "$PROJECT_NAMESPACE" rollout status "deployment/${deployment}" --timeout=180s
|
||||
done
|
||||
fi
|
||||
|
||||
for deployment in $PROJECT_DEPLOYMENTS; do
|
||||
kubectl -n "$PROJECT_NAMESPACE" rollout status "deployment/${deployment}" --timeout=300s
|
||||
done
|
||||
|
||||
DURABLE_K3S_API_URL="$K3S_API_URL"
|
||||
DURABLE_INSECURE_SKIP_TLS_VERIFY=0
|
||||
if [[ "$USE_SSH_TUNNEL_FOR_K3S" == "1" ]]; then
|
||||
|
|
@ -556,6 +621,7 @@ echo "ci_kubeconfig=$CI_KUBECONFIG_PATH"
|
|||
echo "bootstrap_delivery_mode=$BOOTSTRAP_DELIVERY_MODE"
|
||||
echo "forgejo_url=$FORGEJO_ROOT_URL"
|
||||
echo "forgejo_repo=${FORGEJO_ROOT_URL%/}/$FORGEJO_REPO_OWNER/$FORGEJO_REPO_NAME"
|
||||
echo "app_repo=${FORGEJO_ROOT_URL%/}/$APP_FORGEJO_REPO_OWNER/$APP_FORGEJO_REPO_NAME"
|
||||
echo "registry_url=https://$REGISTRY_DOMAIN"
|
||||
echo "grafana_url=$GRAFANA_ROOT_URL"
|
||||
echo "headlamp_url=https://$HEADLAMP_DOMAIN/"
|
||||
|
|
|
|||
|
|
@ -11,14 +11,20 @@ resolve_secret_var FORGEJO_ADMIN_PASSWORD required
|
|||
: "${FORGEJO_ROOT_URL:?set FORGEJO_ROOT_URL}"
|
||||
: "${FORGEJO_PUSH_URL_BASE:=$FORGEJO_ROOT_URL}"
|
||||
: "${FORGEJO_ADMIN_USERNAME:?set FORGEJO_ADMIN_USERNAME}"
|
||||
: "${SOURCE_REPO_DIR:=$ROOT_DIR}"
|
||||
: "${FORGEJO_REPO_OWNER:=$FORGEJO_ADMIN_USERNAME}"
|
||||
: "${FORGEJO_REPO_NAME:=$(basename "$ROOT_DIR")}"
|
||||
: "${FORGEJO_REPO_NAME:=$(basename "$SOURCE_REPO_DIR")}"
|
||||
: "${FORGEJO_PUSH_REMOTE_NAME:=forgejo}"
|
||||
: "${FORGEJO_PUSH_REF:=HEAD:refs/heads/main}"
|
||||
: "${FORGEJO_REPO_HTTP_USERNAME:=$FORGEJO_ADMIN_USERNAME}"
|
||||
|
||||
require git
|
||||
|
||||
if [[ ! -d "$SOURCE_REPO_DIR/.git" ]]; then
|
||||
echo "SOURCE_REPO_DIR is not a git repository: $SOURCE_REPO_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
urlencode() {
|
||||
python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$1"
|
||||
}
|
||||
|
|
@ -33,11 +39,11 @@ if [[ -n "${FORGEJO_ADMIN_PASSWORD:-}" ]]; then
|
|||
auth_remote_url="${auth_remote_url/http:\/\//http://${encoded_username}:${encoded_password}@}"
|
||||
auth_remote_url+="/${FORGEJO_REPO_OWNER}/${FORGEJO_REPO_NAME}.git"
|
||||
fi
|
||||
current_remote_url="$(git remote get-url "$FORGEJO_PUSH_REMOTE_NAME" 2>/dev/null || true)"
|
||||
current_remote_url="$(git -C "$SOURCE_REPO_DIR" remote get-url "$FORGEJO_PUSH_REMOTE_NAME" 2>/dev/null || true)"
|
||||
if [[ -z "$current_remote_url" ]]; then
|
||||
git remote add "$FORGEJO_PUSH_REMOTE_NAME" "$auth_remote_url"
|
||||
git -C "$SOURCE_REPO_DIR" remote add "$FORGEJO_PUSH_REMOTE_NAME" "$auth_remote_url"
|
||||
elif [[ "$current_remote_url" != "$auth_remote_url" ]]; then
|
||||
git remote set-url "$FORGEJO_PUSH_REMOTE_NAME" "$auth_remote_url"
|
||||
git -C "$SOURCE_REPO_DIR" remote set-url "$FORGEJO_PUSH_REMOTE_NAME" "$auth_remote_url"
|
||||
fi
|
||||
|
||||
askpass_script="$(mktemp)"
|
||||
|
|
@ -56,6 +62,6 @@ GIT_TERMINAL_PROMPT=0 \
|
|||
GIT_ASKPASS="$askpass_script" \
|
||||
FORGEJO_ADMIN_USERNAME="$FORGEJO_ADMIN_USERNAME" \
|
||||
FORGEJO_ADMIN_PASSWORD="$FORGEJO_ADMIN_PASSWORD" \
|
||||
git push "$FORGEJO_PUSH_REMOTE_NAME" "$FORGEJO_PUSH_REF"
|
||||
git -C "$SOURCE_REPO_DIR" push "$FORGEJO_PUSH_REMOTE_NAME" "$FORGEJO_PUSH_REF"
|
||||
|
||||
echo "seeded ${remote_url}"
|
||||
echo "seeded ${remote_url} from ${SOURCE_REPO_DIR}"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue