forust
7cca330438
Read traefik logs from Loki instead of file tail. Static machine identity via pre-created secret stops 403 register races. Add janitor cronjob, dashboards, whitelists and metrics.
196 lines
8.2 KiB
YAML
196 lines
8.2 KiB
YAML
# CrowdSec self-healing: static machine identity + enforcement loops.
|
|
#
|
|
# Problem it fixes: the chart's agent init container runs
|
|
# `cscli lapi register --machine "$POD_NAME" ...`
|
|
# unconditionally. Credentials live in an emptyDir, the machine row lives
|
|
# in LAPI's persistent DB. Any init re-run for an already-known pod name
|
|
# (kubelet restart, node reboot) dies with
|
|
# 403 Forbidden: user '<pod>' already exist
|
|
# and the DaemonSet pod sticks in Init forever. Every DS restart also
|
|
# leaves an orphan machine row that is never cleaned.
|
|
#
|
|
# Design (name-independent):
|
|
# * Agent identity is a STATIC machine `crowdsec-agent-workstation`
|
|
# whose password lives in Secret `crowdsec-agent-credentials`
|
|
# (created once, manually - like all other secrets in this repo).
|
|
# The secret is mounted into agent pods at
|
|
# /tmp_config/local_api_credentials.yaml (see extraVolumeMounts in
|
|
# crowdsec-values.yaml), which is exactly the path the agent's main
|
|
# container copies into place at startup.
|
|
# * The DS init command is patched (strategic merge, by container name)
|
|
# to SKIP registration when that file exists, keeping the legacy
|
|
# register path only as fallback. Detection marker in the patched
|
|
# command: `[ -s /tmp_config`.
|
|
# * This CronJob enforces the desired state hourly, so recovery is
|
|
# automatic even after `helm upgrade` reverts the DS patch or the
|
|
# LAPI database is wiped:
|
|
# 1. patch DS init if it still has the unconditional register
|
|
# (no-op otherwise - no restart churn);
|
|
# 2. prune machines with no heartbeat for 2h (orphan hygiene);
|
|
# 3. ensure the static machine exists, recreating it with the
|
|
# Secret password if missing (agent retry loops reconnect
|
|
# on their own - same name + same password);
|
|
# 4. prune bouncer entries idle for 30d.
|
|
#
|
|
# Manual apply (crowdsec/k8s is NOT managed by deploy.yaml):
|
|
# kubectl apply -f crowdsec/k8s/janitor-cronjob.yaml
|
|
# Force a run:
|
|
# kubectl create job -n crowdsec --from=cronjob/crowdsec-janitor janitor-now
|
|
#
|
|
# Helm upgrades: the janitor's strategic patch puts the DS field under
|
|
# the `kubectl-patch` field manager, so a plain `helm upgrade` FAILS
|
|
# with an SSA conflict on initContainers[].command. Procedure:
|
|
# 1. revert init to chart state (kills the conflict):
|
|
# helm template crowdsec crowdsec/crowdsec --version <ver> \
|
|
# -n crowdsec -f crowdsec/k8s/crowdsec-values.yaml > /tmp/r.yaml
|
|
# python3 -c "import yaml,json; ..." # build revert patch from
|
|
# the rendered DaemonSet init command, then
|
|
# kubectl patch ds crowdsec-agent -n crowdsec \
|
|
# --type strategic -p "\$(cat /tmp/revert_patch.json)"
|
|
# 2. helm upgrade --install crowdsec ... (no --force needed)
|
|
# 3. janitor-now right away (upgrade reverts init; new pods would
|
|
# sit in Init until the next hourly run otherwise).
|
|
#
|
|
# One-time bootstrap (order matters):
|
|
# 1. Create Secret + static machine (see commands in chat).
|
|
# 2. Apply this file, trigger janitor-now, wait for agent 1/1.
|
|
# 3. One-time orphan cleanup:
|
|
# kubectl exec -n crowdsec deploy/crowdsec-lapi -- \
|
|
# cscli machines prune --duration 1h --force
|
|
# 4. Only then `helm upgrade` crowdsec with the extraVolumes values.
|
|
# Upgrade reverts the DS patch; trigger janitor-now right after it
|
|
# (otherwise new pods sit in Init until the next hourly run, then
|
|
# self-heal anyway).
|
|
#
|
|
# Password rotation: update the Secret, delete the machine
|
|
# (`cscli machines delete crowdsec-agent-workstation`), trigger
|
|
# janitor-now (recreates it), then `kubectl rollout restart
|
|
# ds/crowdsec-agent -n crowdsec` (agent reads the file at startup only).
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: crowdsec-janitor
|
|
namespace: crowdsec
|
|
labels:
|
|
app.kubernetes.io/part-of: crowdsec
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: crowdsec-janitor
|
|
namespace: crowdsec
|
|
labels:
|
|
app.kubernetes.io/part-of: crowdsec
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["pods"]
|
|
verbs: ["get", "list"]
|
|
- apiGroups: [""]
|
|
resources: ["pods/exec"]
|
|
verbs: ["create"]
|
|
- apiGroups: ["apps"]
|
|
resources: ["daemonsets"]
|
|
verbs: ["get", "patch"]
|
|
# `kubectl exec deploy/<name>` resolves deploy -> replicaset -> pod,
|
|
# which needs read access to these (exec itself is pods/exec above).
|
|
- apiGroups: ["apps"]
|
|
resources: ["deployments", "replicasets"]
|
|
verbs: ["get", "list"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: crowdsec-janitor
|
|
namespace: crowdsec
|
|
labels:
|
|
app.kubernetes.io/part-of: crowdsec
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: crowdsec-janitor
|
|
namespace: crowdsec
|
|
roleRef:
|
|
kind: Role
|
|
name: crowdsec-janitor
|
|
apiGroup: rbac.authorization.k8s.io
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: crowdsec-janitor
|
|
namespace: crowdsec
|
|
labels:
|
|
app.kubernetes.io/part-of: crowdsec
|
|
spec:
|
|
schedule: "17 * * * *"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
activeDeadlineSeconds: 300
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/part-of: crowdsec
|
|
spec:
|
|
serviceAccountName: crowdsec-janitor
|
|
restartPolicy: OnFailure
|
|
containers:
|
|
- name: janitor
|
|
# Same image the chart itself uses for registration jobs;
|
|
# IfNotPresent so it works while the node is offline
|
|
# (layer cached from the chart install).
|
|
image: alpine/kubectl:latest
|
|
imagePullPolicy: IfNotPresent
|
|
env:
|
|
- name: AGENT_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: crowdsec-agent-credentials
|
|
key: password
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -eu
|
|
LAPI_EXEC="kubectl exec -n crowdsec deploy/crowdsec-lapi --"
|
|
echo "== 1. enforce patched agent init =="
|
|
CUR=$(kubectl get ds crowdsec-agent -n crowdsec \
|
|
-o jsonpath='{.spec.template.spec.initContainers[0].command[2]}')
|
|
case "$CUR" in
|
|
*'-s /tmp_config'*)
|
|
echo "init already patched"
|
|
;;
|
|
*)
|
|
echo "patching init"
|
|
WAIT='until nc "$LAPI_HOST" "$LAPI_PORT" -z'
|
|
WAIT="$WAIT; do echo waiting for lapi to start; sleep 5; done"
|
|
LINK='ln -s /staging/etc/crowdsec /etc/crowdsec'
|
|
REG='cscli lapi register --machine "$USERNAME"'
|
|
REG="$REG -u \"\$LAPI_URL\" --token \"\$REGISTRATION_TOKEN\""
|
|
CREDS=/tmp_config/local_api_credentials.yaml
|
|
CMD="$WAIT; $LINK; [ -s $CREDS ] || {"
|
|
CMD="$CMD $REG && cp"
|
|
CMD="$CMD /etc/crowdsec/local_api_credentials.yaml $CREDS; }"
|
|
ESC=$(printf '%s' "$CMD" | sed 's/"/\\"/g')
|
|
PATCH='{"spec":{"template":{"spec":{"initContainers":'
|
|
PATCH=$PATCH'[{"name":"wait-for-lapi-and-register",'
|
|
PATCH=$PATCH'"command":["sh","-c","'$ESC'"]}]}}}}'
|
|
kubectl patch ds crowdsec-agent -n crowdsec \
|
|
--type strategic -p "$PATCH"
|
|
;;
|
|
esac
|
|
echo "== 2. prune orphan machines (no heartbeat for 2h) =="
|
|
$LAPI_EXEC cscli machines prune --duration 2h --force
|
|
echo "== 3. ensure static machine exists =="
|
|
if $LAPI_EXEC cscli machines inspect \
|
|
crowdsec-agent-workstation >/dev/null 2>&1; then
|
|
echo "static machine present"
|
|
else
|
|
echo "recreating static machine"
|
|
$LAPI_EXEC cscli machines add crowdsec-agent-workstation \
|
|
--password "$AGENT_PASSWORD" --force
|
|
fi
|
|
echo "== 4. prune stale bouncers (no pull for 30d) =="
|
|
$LAPI_EXEC cscli bouncers prune -d 720h --force
|