forust
46c7e99b1d
Deploy workflow uses git-tracked manifests, DISABLED flag and kustomize overlays; add webinar-checker metrics with ServiceMonitor and alerts; upgrade shared postgres to 17 with statuspage DB and probes/resources.
305 lines
11 KiB
YAML
305 lines
11 KiB
YAML
name: deploy
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: deploy-main
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
redeploy:
|
|
runs-on: [self-hosted, linux, arch, homelab, prod]
|
|
steps:
|
|
- name: Redeploy workstation
|
|
shell: bash
|
|
env:
|
|
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
|
|
DEPLOY_PORT: ${{ secrets.DEPLOY_PORT }}
|
|
DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
|
|
DEPLOY_PATH: ${{ secrets.DEPLOY_PATH }}
|
|
DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
|
APPLY_PRUNE: ${{ vars.APPLY_PRUNE }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
: "${DEPLOY_HOST:?missing DEPLOY_HOST}"
|
|
: "${DEPLOY_USER:?missing DEPLOY_USER}"
|
|
: "${DEPLOY_KEY:?missing DEPLOY_SSH_KEY}"
|
|
|
|
deploy_port="${DEPLOY_PORT:-22}"
|
|
deploy_path="${DEPLOY_PATH:-/srv/homelab}"
|
|
|
|
ssh_key="$RUNNER_TEMP/deploy_key"
|
|
mkdir -p "$RUNNER_TEMP"
|
|
printf '%s\n' "$DEPLOY_KEY" > "$ssh_key"
|
|
chmod 600 "$ssh_key"
|
|
|
|
ssh_opts=(
|
|
-i "$ssh_key"
|
|
-p "$deploy_port"
|
|
-o BatchMode=yes
|
|
-o StrictHostKeyChecking=accept-new
|
|
)
|
|
|
|
ssh "${ssh_opts[@]}" "${DEPLOY_USER}@${DEPLOY_HOST}" \
|
|
"DEPLOY_PATH=$(printf '%q' \"$deploy_path\") APPLY_PRUNE=$(printf '%q' \"${APPLY_PRUNE:-false}\") bash -se" <<'EOF'
|
|
set -euo pipefail
|
|
|
|
repo="${DEPLOY_PATH:-/srv/homelab}"
|
|
|
|
if [ ! -d "$repo/.git" ]; then
|
|
echo "Repository not found at $repo"
|
|
exit 1
|
|
fi
|
|
|
|
git -C "$repo" fetch origin main
|
|
|
|
echo "== Workstation state =="
|
|
echo " local: $(git -C "$repo" rev-parse --short HEAD)"
|
|
echo " remote: $(git -C "$repo" rev-parse --short origin/main)"
|
|
|
|
if [ -n "$(git -C "$repo" status --porcelain --untracked-files=no)" ]; then
|
|
echo "ERROR: workstation has local tracked modifications, refusing reset:"
|
|
git -C "$repo" status --porcelain --untracked-files=no
|
|
git -C "$repo" diff --stat
|
|
exit 1
|
|
fi
|
|
|
|
git -C "$repo" reset --hard origin/main
|
|
cd "$repo"
|
|
|
|
is_disabled() {
|
|
local target="$1"
|
|
if [ -f "$target" ]; then
|
|
target="$(dirname "$target")"
|
|
fi
|
|
while true; do
|
|
if [ -f "$target/DISABLED" ]; then
|
|
return 0
|
|
fi
|
|
if [ "$target" = "$repo" ]; then
|
|
break
|
|
fi
|
|
target="$(dirname "$target")"
|
|
case "$target" in
|
|
"$repo"/*) ;;
|
|
*) break ;;
|
|
esac
|
|
done
|
|
return 1
|
|
}
|
|
|
|
collect_k8s() {
|
|
git ls-files -- "$1" \
|
|
| grep -E '\.ya?ml$' \
|
|
| grep -Ev '/routing/|/overlays/' \
|
|
| grep -Ev '(^|/)(kustomization\.ya?ml|.*\.example\.ya?ml|.*values\.ya?ml|patch-.*\.ya?ml)$' \
|
|
| grep -Ev '(^|/)[^/]*secret[^/]*\.ya?ml$' \
|
|
| sort
|
|
}
|
|
|
|
collect_k8s_inactive() {
|
|
collect_k8s "$1" \
|
|
| grep -E '(^|/)namespace\.ya?ml$|/routing/'
|
|
}
|
|
|
|
kustomize_overlay() {
|
|
if [ -f "$1/overlays/prod/kustomization.yaml" ]; then
|
|
echo "$1/overlays/prod"
|
|
elif [ -f "$1/base/kustomization.yaml" ]; then
|
|
echo "$1/base"
|
|
fi
|
|
}
|
|
|
|
mapfile -t k8s_dirs < <(
|
|
git ls-files '*.yaml' '*.yml' \
|
|
| grep -E '(^|/)k8s/' \
|
|
| sed -E 's#((^|.*/)k8s)/.*#\1#' \
|
|
| sort -u
|
|
)
|
|
|
|
k8s_manifests=()
|
|
kustomize_apps=()
|
|
for kd_rel in "${k8s_dirs[@]}"; do
|
|
kd="$repo/$kd_rel"
|
|
if is_disabled "$kd"; then
|
|
echo "skip (DISABLED): $kd_rel"
|
|
continue
|
|
fi
|
|
if [ -f "$kd/active" ]; then
|
|
overlay="$(kustomize_overlay "$kd" || true)"
|
|
if [ -n "${overlay:-}" ]; then
|
|
echo "kustomize app: ${overlay#$repo/}"
|
|
kustomize_apps+=("$overlay")
|
|
else
|
|
while IFS= read -r f; do
|
|
[ -n "$f" ] && k8s_manifests+=("$repo/$f")
|
|
done < <(collect_k8s "$kd_rel" || true)
|
|
fi
|
|
else
|
|
while IFS= read -r f; do
|
|
[ -n "$f" ] && k8s_manifests+=("$repo/$f")
|
|
done < <(collect_k8s_inactive "$kd_rel" || true)
|
|
fi
|
|
done
|
|
|
|
mapfile -t compose_rel < <(
|
|
git ls-files '*/compose.yaml' '*/compose.yml' compose.yaml compose.yml | sort
|
|
)
|
|
|
|
compose_stacks=()
|
|
for cf_rel in "${compose_rel[@]}"; do
|
|
cf="$repo/$cf_rel"
|
|
if is_disabled "$cf"; then
|
|
echo "skip (DISABLED): $cf_rel"
|
|
continue
|
|
fi
|
|
if [ -f "$(dirname "$cf")/k8s/active" ]; then
|
|
echo "skip (k8s-managed): $cf_rel"
|
|
continue
|
|
fi
|
|
compose_stacks+=("$cf")
|
|
done
|
|
|
|
echo "== Validate compose stacks =="
|
|
for cf in "${compose_stacks[@]}"; do
|
|
echo " config: $cf"
|
|
docker compose -f "$cf" config --quiet
|
|
done
|
|
|
|
echo "== Validate k8s manifests (kubectl dry-run=client) =="
|
|
for m in "${k8s_manifests[@]}"; do
|
|
echo " apply --dry-run=client $m"
|
|
kubectl apply --dry-run=client -f "$m" >/dev/null
|
|
done
|
|
for k in "${kustomize_apps[@]}"; do
|
|
echo " apply -k --dry-run=client $k"
|
|
kubectl apply -k "$k" --dry-run=client >/dev/null
|
|
done
|
|
|
|
echo "== Validate k8s manifests (kubectl dry-run=server) =="
|
|
for m in "${k8s_manifests[@]}"; do
|
|
echo " apply --dry-run=server $m"
|
|
kubectl apply --dry-run=server -f "$m" >/dev/null
|
|
done
|
|
for k in "${kustomize_apps[@]}"; do
|
|
echo " apply -k --dry-run=server $k"
|
|
kubectl apply -k "$k" --dry-run=server >/dev/null
|
|
done
|
|
|
|
echo "== Checking referenced Secrets exist =="
|
|
echo " (deploy never applies *secret*.yaml; create missing ones from the laptop)"
|
|
ref_secrets=()
|
|
if [ "${#k8s_manifests[@]}" -gt 0 ]; then
|
|
while IFS= read -r s; do
|
|
[ -n "$s" ] && ref_secrets+=("$s")
|
|
done < <(
|
|
{
|
|
grep -h -A1 -E 'secretRef:|secretKeyRef:' "${k8s_manifests[@]}" 2>/dev/null || true
|
|
grep -h -E 'secretName:' "${k8s_manifests[@]}" 2>/dev/null || true
|
|
} | grep -E 'name:' | sed -E 's/.*name:[[:space:]]*//' | tr -d '"'"'"' "'"'" | sed -E 's/[[:space:]]*#.*//' | awk 'NF' | sort -u || true
|
|
)
|
|
fi
|
|
missing_secrets=()
|
|
all_secrets="$(kubectl get secrets -A --no-headers -o custom-columns=:metadata.name 2>/dev/null || true)"
|
|
for s in "${ref_secrets[@]}"; do
|
|
if printf '%s\n' "$all_secrets" | grep -qx "$s"; then
|
|
echo " ok: $s"
|
|
else
|
|
echo " MISSING: $s"
|
|
missing_secrets+=("$s")
|
|
fi
|
|
done
|
|
if [ "${#missing_secrets[@]}" -gt 0 ]; then
|
|
echo "ERROR: ${#missing_secrets[@]} referenced Secret(s) not found in the cluster:"
|
|
printf ' - %s\n' "${missing_secrets[@]}"
|
|
echo "Create them manually from the laptop, e.g.:"
|
|
echo " kubectl apply -f SERVICE/k8s/secrets.yaml # see SERVICE/k8s/secrets.yaml.example"
|
|
exit 1
|
|
fi
|
|
|
|
echo "== Applying Kubernetes manifests =="
|
|
ns_files=()
|
|
other_files=()
|
|
for m in "${k8s_manifests[@]}"; do
|
|
case "$m" in
|
|
*/namespace.y?ml) ns_files+=("$m") ;;
|
|
*) other_files+=("$m") ;;
|
|
esac
|
|
done
|
|
|
|
prune_opts=()
|
|
if [ "${APPLY_PRUNE:-false}" = "true" ]; then
|
|
prune_opts=(--prune -l app.kubernetes.io/managed-by=homelab-deploy)
|
|
fi
|
|
|
|
if [ "${#ns_files[@]}" -gt 0 ]; then
|
|
echo " namespaces first: ${ns_files[*]}"
|
|
kubectl apply -f "${ns_files[@]}"
|
|
fi
|
|
if [ -f "$repo/prometheus-stack/k8s/active" ] && ! is_disabled "$repo/prometheus-stack/k8s"; then
|
|
if [ ! -f "$repo/prometheus-stack/k8s/grafana-values.yaml" ]; then
|
|
echo "ERROR: prometheus-stack/k8s/grafana-values.yaml (gitignored) missing on workstation, restore it first."
|
|
exit 1
|
|
fi
|
|
echo "== Upgrading kube-prometheus-stack =="
|
|
helm upgrade --install prometheus-stack prometheus-community/kube-prometheus-stack \
|
|
--namespace prometheus \
|
|
--version 86.2.3 \
|
|
--values "$repo/prometheus-stack/k8s/grafana-values.yaml" \
|
|
--wait --timeout 10m
|
|
fi
|
|
if [ -f "$repo/loki/k8s/active" ] && ! is_disabled "$repo/loki/k8s"; then
|
|
echo "== Upgrading loki/alloy =="
|
|
helm repo add grafana https://grafana.github.io/helm-charts >/dev/null 2>&1 || true
|
|
helm repo update grafana >/dev/null 2>&1 || true
|
|
helm upgrade --install loki grafana/loki \
|
|
--version 7.3.0 \
|
|
--namespace prometheus \
|
|
--values "$repo/loki/k8s/loki-values.yaml" \
|
|
--wait --timeout 10m
|
|
helm upgrade --install alloy grafana/alloy \
|
|
--version 1.12.1 \
|
|
--namespace prometheus \
|
|
--values "$repo/loki/k8s/alloy-values.yaml" \
|
|
--wait --timeout 10m
|
|
fi
|
|
|
|
if [ "${#other_files[@]}" -gt 0 ]; then
|
|
echo " resources: ${other_files[*]}"
|
|
kubectl apply "${prune_opts[@]}" -f "${other_files[@]}"
|
|
fi
|
|
|
|
for k in "${kustomize_apps[@]}"; do
|
|
echo "== Applying kustomize app: ${k#$repo/} =="
|
|
kubectl apply -k "$k"
|
|
done
|
|
|
|
if [ -f "$repo/userbot/k8s/active" ] && ! is_disabled "$repo/userbot"; then
|
|
echo "== userbot panel hook =="
|
|
if kubectl get secret userbot-common-secrets -n userbot >/dev/null 2>&1; then
|
|
echo " userbot-common-secrets already present in userbot ns, not touching"
|
|
elif kubectl get secret userbot-common-secrets -n default >/dev/null 2>&1; then
|
|
echo " bootstrapping userbot-common-secrets into userbot ns"
|
|
kubectl get secret userbot-common-secrets -n default -o json \
|
|
| jq 'del(.metadata.annotations,.metadata.creationTimestamp,.metadata.resourceVersion,.metadata.uid,.metadata.managedFields) | .metadata.namespace = "userbot"' \
|
|
| kubectl apply -f -
|
|
else
|
|
echo " WARNING: userbot-common-secrets missing in both default and userbot ns; create it manually from the laptop"
|
|
fi
|
|
kubectl rollout restart deployment/userbot-panel -n userbot
|
|
kubectl rollout status deployment/userbot-panel -n userbot --timeout=180s
|
|
fi
|
|
|
|
echo "== Redeploying docker compose stacks =="
|
|
for cf in "${compose_stacks[@]}"; do
|
|
echo " compose: $cf"
|
|
if grep -Eq '^\s+pull_policy:\s*build\b' "$cf"; then
|
|
docker compose -f "$cf" build
|
|
docker compose -f "$cf" push
|
|
fi
|
|
docker compose -f "$cf" up -d --pull always --remove-orphans
|
|
done
|
|
EOF
|