feat(crowdsec): switch agent to loki acquisition with static identity
Read traefik logs from Loki instead of file tail. Static machine identity via pre-created secret stops 403 register races. Add janitor cronjob, dashboards, whitelists and metrics.
This commit is contained in:
forust committed
1 parent
5936de3e56
commit
7cca330438
4 files changed
+314
-35
No files matched your search
@@ -0,0 +1,195 @@
|
||||
# CrowdSec self-healing: static machine identity + enforcement loops.
|
||||
#
|
||||
# Problem it fixes: the chart's agent init container runs
|
||||
# `cscli lapi register --machine "$POD_NAME" ...`
|
||||
# unconditionally. Credentials live in an emptyDir, the machine row lives
|
||||
# in LAPI's persistent DB. Any init re-run for an already-known pod name
|
||||
# (kubelet restart, node reboot) dies with
|
||||
# 403 Forbidden: user '<pod>' already exist
|
||||
# and the DaemonSet pod sticks in Init forever. Every DS restart also
|
||||
# leaves an orphan machine row that is never cleaned.
|
||||
#
|
||||
# Design (name-independent):
|
||||
# * Agent identity is a STATIC machine `crowdsec-agent-workstation`
|
||||
# whose password lives in Secret `crowdsec-agent-credentials`
|
||||
# (created once, manually - like all other secrets in this repo).
|
||||
# The secret is mounted into agent pods at
|
||||
# /tmp_config/local_api_credentials.yaml (see extraVolumeMounts in
|
||||
# crowdsec-values.yaml), which is exactly the path the agent's main
|
||||
# container copies into place at startup.
|
||||
# * The DS init command is patched (strategic merge, by container name)
|
||||
# to SKIP registration when that file exists, keeping the legacy
|
||||
# register path only as fallback. Detection marker in the patched
|
||||
# command: `[ -s /tmp_config`.
|
||||
# * This CronJob enforces the desired state hourly, so recovery is
|
||||
# automatic even after `helm upgrade` reverts the DS patch or the
|
||||
# LAPI database is wiped:
|
||||
# 1. patch DS init if it still has the unconditional register
|
||||
# (no-op otherwise - no restart churn);
|
||||
# 2. prune machines with no heartbeat for 2h (orphan hygiene);
|
||||
# 3. ensure the static machine exists, recreating it with the
|
||||
# Secret password if missing (agent retry loops reconnect
|
||||
# on their own - same name + same password);
|
||||
# 4. prune bouncer entries idle for 30d.
|
||||
#
|
||||
# Manual apply (crowdsec/k8s is NOT managed by deploy.yaml):
|
||||
# kubectl apply -f crowdsec/k8s/janitor-cronjob.yaml
|
||||
# Force a run:
|
||||
# kubectl create job -n crowdsec --from=cronjob/crowdsec-janitor janitor-now
|
||||
#
|
||||
# Helm upgrades: the janitor's strategic patch puts the DS field under
|
||||
# the `kubectl-patch` field manager, so a plain `helm upgrade` FAILS
|
||||
# with an SSA conflict on initContainers[].command. Procedure:
|
||||
# 1. revert init to chart state (kills the conflict):
|
||||
# helm template crowdsec crowdsec/crowdsec --version <ver> \
|
||||
# -n crowdsec -f crowdsec/k8s/crowdsec-values.yaml > /tmp/r.yaml
|
||||
# python3 -c "import yaml,json; ..." # build revert patch from
|
||||
# the rendered DaemonSet init command, then
|
||||
# kubectl patch ds crowdsec-agent -n crowdsec \
|
||||
# --type strategic -p "\$(cat /tmp/revert_patch.json)"
|
||||
# 2. helm upgrade --install crowdsec ... (no --force needed)
|
||||
# 3. janitor-now right away (upgrade reverts init; new pods would
|
||||
# sit in Init until the next hourly run otherwise).
|
||||
#
|
||||
# One-time bootstrap (order matters):
|
||||
# 1. Create Secret + static machine (see commands in chat).
|
||||
# 2. Apply this file, trigger janitor-now, wait for agent 1/1.
|
||||
# 3. One-time orphan cleanup:
|
||||
# kubectl exec -n crowdsec deploy/crowdsec-lapi -- \
|
||||
# cscli machines prune --duration 1h --force
|
||||
# 4. Only then `helm upgrade` crowdsec with the extraVolumes values.
|
||||
# Upgrade reverts the DS patch; trigger janitor-now right after it
|
||||
# (otherwise new pods sit in Init until the next hourly run, then
|
||||
# self-heal anyway).
|
||||
#
|
||||
# Password rotation: update the Secret, delete the machine
|
||||
# (`cscli machines delete crowdsec-agent-workstation`), trigger
|
||||
# janitor-now (recreates it), then `kubectl rollout restart
|
||||
# ds/crowdsec-agent -n crowdsec` (agent reads the file at startup only).
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: crowdsec-janitor
|
||||
namespace: crowdsec
|
||||
labels:
|
||||
app.kubernetes.io/part-of: crowdsec
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: crowdsec-janitor
|
||||
namespace: crowdsec
|
||||
labels:
|
||||
app.kubernetes.io/part-of: crowdsec
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["daemonsets"]
|
||||
verbs: ["get", "patch"]
|
||||
# `kubectl exec deploy/<name>` resolves deploy -> replicaset -> pod,
|
||||
# which needs read access to these (exec itself is pods/exec above).
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments", "replicasets"]
|
||||
verbs: ["get", "list"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: crowdsec-janitor
|
||||
namespace: crowdsec
|
||||
labels:
|
||||
app.kubernetes.io/part-of: crowdsec
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: crowdsec-janitor
|
||||
namespace: crowdsec
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: crowdsec-janitor
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: crowdsec-janitor
|
||||
namespace: crowdsec
|
||||
labels:
|
||||
app.kubernetes.io/part-of: crowdsec
|
||||
spec:
|
||||
schedule: "17 * * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
activeDeadlineSeconds: 300
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/part-of: crowdsec
|
||||
spec:
|
||||
serviceAccountName: crowdsec-janitor
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: janitor
|
||||
# Same image the chart itself uses for registration jobs;
|
||||
# IfNotPresent so it works while the node is offline
|
||||
# (layer cached from the chart install).
|
||||
image: alpine/kubectl:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: AGENT_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: crowdsec-agent-credentials
|
||||
key: password
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
LAPI_EXEC="kubectl exec -n crowdsec deploy/crowdsec-lapi --"
|
||||
echo "== 1. enforce patched agent init =="
|
||||
CUR=$(kubectl get ds crowdsec-agent -n crowdsec \
|
||||
-o jsonpath='{.spec.template.spec.initContainers[0].command[2]}')
|
||||
case "$CUR" in
|
||||
*'-s /tmp_config'*)
|
||||
echo "init already patched"
|
||||
;;
|
||||
*)
|
||||
echo "patching init"
|
||||
WAIT='until nc "$LAPI_HOST" "$LAPI_PORT" -z'
|
||||
WAIT="$WAIT; do echo waiting for lapi to start; sleep 5; done"
|
||||
LINK='ln -s /staging/etc/crowdsec /etc/crowdsec'
|
||||
REG='cscli lapi register --machine "$USERNAME"'
|
||||
REG="$REG -u \"\$LAPI_URL\" --token \"\$REGISTRATION_TOKEN\""
|
||||
CREDS=/tmp_config/local_api_credentials.yaml
|
||||
CMD="$WAIT; $LINK; [ -s $CREDS ] || {"
|
||||
CMD="$CMD $REG && cp"
|
||||
CMD="$CMD /etc/crowdsec/local_api_credentials.yaml $CREDS; }"
|
||||
ESC=$(printf '%s' "$CMD" | sed 's/"/\\"/g')
|
||||
PATCH='{"spec":{"template":{"spec":{"initContainers":'
|
||||
PATCH=$PATCH'[{"name":"wait-for-lapi-and-register",'
|
||||
PATCH=$PATCH'"command":["sh","-c","'$ESC'"]}]}}}}'
|
||||
kubectl patch ds crowdsec-agent -n crowdsec \
|
||||
--type strategic -p "$PATCH"
|
||||
;;
|
||||
esac
|
||||
echo "== 2. prune orphan machines (no heartbeat for 2h) =="
|
||||
$LAPI_EXEC cscli machines prune --duration 2h --force
|
||||
echo "== 3. ensure static machine exists =="
|
||||
if $LAPI_EXEC cscli machines inspect \
|
||||
crowdsec-agent-workstation >/dev/null 2>&1; then
|
||||
echo "static machine present"
|
||||
else
|
||||
echo "recreating static machine"
|
||||
$LAPI_EXEC cscli machines add crowdsec-agent-workstation \
|
||||
--password "$AGENT_PASSWORD" --force
|
||||
fi
|
||||
echo "== 4. prune stale bouncers (no pull for 30d) =="
|
||||
$LAPI_EXEC cscli bouncers prune -d 720h --force
|
||||
Reference in new issue
Block a user