From 83b2e68371620cfa9fa4d3e00622cfd2ac167322 Mon Sep 17 00:00:00 2001 From: mr-forust Date: Tue, 6 Oct 2026 23:42:33 +0200 Subject: [PATCH] feat(metrics): collect Headscale, NetBird, Gitea and Immich metrics --- gitea/k8s/config.yaml | 2 ++ gitea/k8s/gitea.yaml | 2 ++ gitea/k8s/ingress.yaml | 3 ++- gitea/k8s/servicemonitor.yaml | 16 +++++++++++++ headscale/config/headscale.yaml.example | 2 +- headscale/k8s/routing/external-service.yaml | 2 ++ headscale/k8s/vmservicescrape.yaml | 18 ++++++++++++++ immich/k8s/config.yaml | 4 ++++ immich/k8s/immich.yaml | 12 ++++++++++ immich/k8s/servicemonitor.yaml | 20 ++++++++++++++++ netbird/k8s/netbird.yaml | 9 +++++++ netbird/k8s/servicemonitor.yaml | 16 +++++++++++++ prometheus-stack/k8s/README.md | 26 +++++++++++++++++++++ 13 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 gitea/k8s/servicemonitor.yaml create mode 100644 headscale/k8s/vmservicescrape.yaml create mode 100644 immich/k8s/servicemonitor.yaml create mode 100644 netbird/k8s/servicemonitor.yaml diff --git a/gitea/k8s/config.yaml b/gitea/k8s/config.yaml index 771d5a6..3fc5f41 100644 --- a/gitea/k8s/config.yaml +++ b/gitea/k8s/config.yaml @@ -20,6 +20,8 @@ data: GITEA__mailer__ENABLED: "false" + GITEA__metrics__ENABLED: "true" + # No code/issue search needed: bleve reindexes the whole issue index on # every pod restart (cron.rebuild_issue_indexer RUN_AT_START) and hammers # the rotational disk for an hour. "db" serves issue search from postgres. diff --git a/gitea/k8s/gitea.yaml b/gitea/k8s/gitea.yaml index 1b75c91..0724f54 100644 --- a/gitea/k8s/gitea.yaml +++ b/gitea/k8s/gitea.yaml @@ -3,6 +3,8 @@ kind: Service metadata: name: gitea-service namespace: gitea + labels: + app: gitea spec: selector: app: gitea diff --git a/gitea/k8s/ingress.yaml b/gitea/k8s/ingress.yaml index e39e67c..596bac2 100644 --- a/gitea/k8s/ingress.yaml +++ b/gitea/k8s/ingress.yaml @@ -7,7 +7,8 @@ spec: entryPoints: - websecure routes: - - match: Host(`gitea.forust.xyz`) || Host(`git.forust.xyz`) + # Metrics are scraped directly through the cluster Service. + - match: (Host(`gitea.forust.xyz`) || Host(`git.forust.xyz`)) && !PathPrefix(`/metrics`) kind: Rule services: - name: gitea-service diff --git a/gitea/k8s/servicemonitor.yaml b/gitea/k8s/servicemonitor.yaml new file mode 100644 index 0000000..31e71ae --- /dev/null +++ b/gitea/k8s/servicemonitor.yaml @@ -0,0 +1,16 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: gitea + namespace: gitea + labels: + release: prometheus-stack +spec: + selector: + matchLabels: + app: gitea + endpoints: + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/headscale/config/headscale.yaml.example b/headscale/config/headscale.yaml.example index 6242ccc..6747e27 100644 --- a/headscale/config/headscale.yaml.example +++ b/headscale/config/headscale.yaml.example @@ -7,7 +7,7 @@ # # Dev server_url # server_url: https://hs.dev_internal_domain.internal listen_addr: 0.0.0.0:8080 -metrics_listen_addr: 127.0.0.1:9090 +metrics_listen_addr: 0.0.0.0:9090 grpc_listen_addr: 127.0.0.1:50443 grpc_allow_insecure: false noise: diff --git a/headscale/k8s/routing/external-service.yaml b/headscale/k8s/routing/external-service.yaml index c5e12ec..0e8e4a0 100644 --- a/headscale/k8s/routing/external-service.yaml +++ b/headscale/k8s/routing/external-service.yaml @@ -3,6 +3,8 @@ kind: Service metadata: name: headscale-server-external namespace: headscale + labels: + app: headscale spec: ports: - port: 8080 diff --git a/headscale/k8s/vmservicescrape.yaml b/headscale/k8s/vmservicescrape.yaml new file mode 100644 index 0000000..b2e82e3 --- /dev/null +++ b/headscale/k8s/vmservicescrape.yaml @@ -0,0 +1,18 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMServiceScrape +metadata: + name: headscale + namespace: headscale + labels: + release: prometheus-stack +spec: + # The external Service has a manually managed EndpointSlice, not Endpoints. + discoveryRole: endpointslice + selector: + matchLabels: + app: headscale + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/immich/k8s/config.yaml b/immich/k8s/config.yaml index b8a2fce..c9568d6 100644 --- a/immich/k8s/config.yaml +++ b/immich/k8s/config.yaml @@ -6,6 +6,10 @@ metadata: data: TZ: "Europe/Bratislava" + IMMICH_TELEMETRY_INCLUDE: "all" + IMMICH_API_METRICS_PORT: "8081" + IMMICH_MICROSERVICES_METRICS_PORT: "8082" + # The database in this namespace, not the shared one in the database # namespace: v3 needs VectorChord, and only the dedicated image carries it. DB_HOSTNAME: "immich-postgres" diff --git a/immich/k8s/immich.yaml b/immich/k8s/immich.yaml index 448e5c1..ffb3b7b 100644 --- a/immich/k8s/immich.yaml +++ b/immich/k8s/immich.yaml @@ -3,6 +3,8 @@ kind: Service metadata: name: immich-service namespace: immich + labels: + app: immich spec: selector: app: immich @@ -10,6 +12,12 @@ spec: - name: http port: 2283 targetPort: 2283 + - name: api-metrics + port: 8081 + targetPort: api-metrics + - name: worker-metrics + port: 8082 + targetPort: worker-metrics --- apiVersion: apps/v1 kind: Deployment @@ -41,6 +49,10 @@ spec: ports: - name: http containerPort: 2283 + - name: api-metrics + containerPort: 8081 + - name: worker-metrics + containerPort: 8082 volumeMounts: - name: immich-data mountPath: /data diff --git a/immich/k8s/servicemonitor.yaml b/immich/k8s/servicemonitor.yaml new file mode 100644 index 0000000..bfae350 --- /dev/null +++ b/immich/k8s/servicemonitor.yaml @@ -0,0 +1,20 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: immich + namespace: immich + labels: + release: prometheus-stack +spec: + selector: + matchLabels: + app: immich + endpoints: + - port: api-metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s + - port: worker-metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/netbird/k8s/netbird.yaml b/netbird/k8s/netbird.yaml index bdd8f2f..75e52c0 100644 --- a/netbird/k8s/netbird.yaml +++ b/netbird/k8s/netbird.yaml @@ -3,6 +3,8 @@ kind: Service metadata: name: netbird-server-service namespace: netbird + labels: + app: netbird-server spec: selector: app: netbird-server @@ -11,6 +13,10 @@ spec: name: http targetPort: 80 protocol: TCP + - port: 9090 + name: metrics + targetPort: metrics + protocol: TCP - port: 3478 name: stun targetPort: 3478 @@ -59,6 +65,9 @@ spec: - containerPort: 80 name: http protocol: TCP + - containerPort: 9090 + name: metrics + protocol: TCP - containerPort: 3478 name: stun protocol: UDP diff --git a/netbird/k8s/servicemonitor.yaml b/netbird/k8s/servicemonitor.yaml new file mode 100644 index 0000000..05fac52 --- /dev/null +++ b/netbird/k8s/servicemonitor.yaml @@ -0,0 +1,16 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: netbird-server + namespace: netbird + labels: + release: prometheus-stack +spec: + selector: + matchLabels: + app: netbird-server + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/prometheus-stack/k8s/README.md b/prometheus-stack/k8s/README.md index 494726c..5672e9e 100644 --- a/prometheus-stack/k8s/README.md +++ b/prometheus-stack/k8s/README.md @@ -15,3 +15,29 @@ The VictoriaMetrics Operator chart and its CRDs are installed before the Kubernetes manifests by the normal deploy workflow. On a cluster where the operator CRDs are not installed yet, CI skips the server-side dry-run of the `VMAgent` resource; the deploy installs the chart before applying that resource. + +## Application metrics + +The application ServiceMonitors use a 30s interval and a 10s timeout: + +- Headscale: the external Service points to the Compose host on port 19090. + A VMServiceScrape uses EndpointSlice discovery for this manually managed target. + The Compose configuration must bind metrics to `0.0.0.0:9090`. +- NetBird: the combined server exports `/metrics` on port 9090. The existing + `server.metricsPort` setting enables the listener. +- Gitea: `GITEA__metrics__ENABLED` enables `/metrics` on the HTTP port. The public + ingress excludes this path. The monitor uses the internal Service directly. +- Immich: `IMMICH_TELEMETRY_INCLUDE=all` enables API and worker metrics on ports + 8081 and 8082. The monitor scrapes both ports on each server replica. + +Deploy through the existing CI and deploy workflow. Gitea and Immich reload their +ConfigMap changes through Reloader. Check the VMAgent targets after deployment +and query `up{scraper="victoria",namespace=~"netbird|gitea|immich|headscale"}` in +VictoriaMetrics. All targets should report 1. + +For rollback, revert the application metrics changes, run CI, and deploy the +revert. Remove the three application ServiceMonitors and the Headscale VMServiceScrape explicitly: the deployment +workflow applies manifests and does not prune removed resources. + +For Headscale rollback, remove its VMServiceScrape and Service label, restore the +previous Compose metrics bind address, and restart only the Headscale service.