feat(monitoring): add VictoriaMetrics trial stack

prometheus-stack/k8s/grafana-values.yaml

@@ -38,6 +38,9 @@ grafana:
 
   # One block covers both the dashboards and datasources sidecars (p95 91M / 80M).
   sidecar:
+    datasources:
+      # Chart built-in Prometheus DS disabled: VictoriaMetrics (below) is the default.
+      defaultDatasourceEnabled: false
     resources:
       requests:
         memory: "96Mi"
@@ -50,6 +53,11 @@ grafana:
       type: loki
       url: http://loki-gateway.prometheus.svc.cluster.local
       access: proxy
+    - name: VictoriaMetrics
+      type: prometheus
+      url: http://victoria-metrics.prometheus.svc.cluster.local:8428
+      access: proxy
+      isDefault: true
 
 prometheus:
   prometheusSpec:

prometheus-stack/k8s/ingress.yaml

@@ -31,3 +31,105 @@ spec:
           port: 80
   tls:
     secretName: internal-wildcard-tls
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: prometheus-local
+  namespace: prometheus
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`prom.workstation.internal`) || Host(`prom.gigaforust.internal`)
+      kind: Rule
+      services:
+        - name: prometheus-stack-kube-prom-prometheus
+          port: 9090
+  tls:
+    secretName: internal-wildcard-tls
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: alertmanager-local
+  namespace: prometheus
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`am.workstation.internal`) || Host(`am.gigaforust.internal`)
+      kind: Rule
+      services:
+        - name: prometheus-stack-kube-prom-alertmanager
+          port: 9093
+  tls:
+    secretName: internal-wildcard-tls
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: loki-local
+  namespace: prometheus
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`loki.workstation.internal`) || Host(`loki.gigaforust.internal`)
+      kind: Rule
+      services:
+        - name: loki-gateway
+          port: 80
+  tls:
+    secretName: internal-wildcard-tls
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: alloy-local
+  namespace: prometheus
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`alloy.workstation.internal`) || Host(`alloy.gigaforust.internal`)
+      kind: Rule
+      services:
+        - name: alloy
+          port: 12345
+  tls:
+    secretName: internal-wildcard-tls
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: victoria-local
+  namespace: prometheus
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`victoria.workstation.internal`) || Host(`victoria.gigaforust.internal`)
+      kind: Rule
+      services:
+        - name: victoria-metrics
+          port: 8428
+  tls:
+    secretName: internal-wildcard-tls
+---
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: vmalert-local
+  namespace: prometheus
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`vmalert.workstation.internal`) || Host(`vmalert.gigaforust.internal`)
+      kind: Rule
+      services:
+        - name: vmalert
+          port: 8880
+  tls:
+    secretName: internal-wildcard-tls

prometheus-stack/k8s/victoria-secrets.yaml.example

@@ -0,0 +1,10 @@
+# Create the victoria-secrets Secret before deploying VictoriaMetrics.
+# Copy the metrics password from uptime-kuma/k8s/secrets.yaml into the value below.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: victoria-secrets
+  namespace: prometheus
+type: Opaque
+stringData:
+  uptime-kuma-password: "REPLACE_ME"

prometheus-stack/k8s/victoria.yaml

@@ -0,0 +1,122 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: victoria
+  namespace: prometheus
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: victoria
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-stack-kube-prom-prometheus
+subjects:
+  - kind: ServiceAccount
+    name: victoria
+    namespace: prometheus
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: victoria-metrics
+  namespace: prometheus
+spec:
+  selector:
+    app: victoria-metrics
+  ports:
+    - port: 8428
+      targetPort: 8428
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: victoria-pvc
+  namespace: prometheus
+spec:
+  resources:
+    requests:
+      storage: 10Gi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: victoria-deployment
+  namespace: prometheus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: victoria-metrics
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: victoria-metrics
+    spec:
+      serviceAccountName: victoria
+      containers:
+        - name: victoria
+          image: victoriametrics/victoria-metrics:v1.153.0-scratch
+          args:
+            - -storageDataPath=/vmdata
+            - -retentionPeriod=30d
+            - -httpListenAddr=:8428
+            - -promscrape.config=/etc/vm/conf/scrape.yaml
+            - -promscrape.configCheckInterval=60s
+          ports:
+            - containerPort: 8428
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8428
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8428
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            failureThreshold: 3
+          volumeMounts:
+            - name: vmdata
+              mountPath: /vmdata
+            - name: scrape-config
+              mountPath: /etc/vm/conf
+              readOnly: true
+            - name: vm-secrets
+              mountPath: /etc/vm/secrets
+              readOnly: true
+            - name: prom-admission-ca
+              mountPath: /etc/prometheus/certs
+              readOnly: true
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "256Mi"
+            limits:
+              cpu: "1000m"
+              memory: "1Gi"
+      volumes:
+        - name: vmdata
+          persistentVolumeClaim:
+            claimName: victoria-pvc
+        - name: scrape-config
+          configMap:
+            name: victoria-scrape
+        - name: vm-secrets
+          secret:
+            secretName: victoria-secrets
+        - name: prom-admission-ca
+          secret:
+            secretName: prometheus-stack-kube-prom-admission
+            items:
+              - key: ca
+                path: 0_prometheus_prometheus-stack-kube-prom-admission_ca

prometheus-stack/k8s/vmalert.yaml

@@ -0,0 +1,70 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: vmalert
+  namespace: prometheus
+spec:
+  selector:
+    app: vmalert
+  ports:
+    - port: 8880
+      targetPort: 8880
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: vmalert-deployment
+  namespace: prometheus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: vmalert
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: vmalert
+    spec:
+      containers:
+        - name: vmalert
+          image: victoriametrics/vmalert:v1.153.0
+          args:
+            - -datasource.url=http://victoria-metrics.prometheus.svc.cluster.local:8428
+            - -remoteWrite.url=http://victoria-metrics.prometheus.svc.cluster.local:8428
+            - -notifier.url=http://prometheus-stack-kube-prom-alertmanager.prometheus.svc.cluster.local:9093
+            - -rule=/etc/vm/rules/*.yaml
+            - -evaluationInterval=60s
+            - -httpListenAddr=:8880
+          ports:
+            - containerPort: 8880
+          readinessProbe:
+            httpGet:
+              path: /metrics
+              port: 8880
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /metrics
+              port: 8880
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            failureThreshold: 3
+          volumeMounts:
+            - name: rules
+              mountPath: /etc/vm/rules
+              readOnly: true
+          resources:
+            requests:
+              cpu: "50m"
+              memory: "64Mi"
+            limits:
+              cpu: "200m"
+              memory: "256Mi"
+      volumes:
+        - name: rules
+          configMap:
+            name: prometheus-prometheus-stack-kube-prom-prometheus-rulefiles-0

prometheus-stack/k8s/vmctl-backfill.yaml

@@ -0,0 +1,31 @@
+# One-shot history backfill Prometheus -> VictoriaMetrics via vmctl remote-read.
+# Safe to keep applied: a completed Job is a no-op on re-apply.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: vmctl-backfill
+  namespace: prometheus
+spec:
+  backoffLimit: 2
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: vmctl
+          image: victoriametrics/vmctl:v1.153.0
+          args:
+            - remote-read
+            - -s
+            - --disable-progress-bar
+            - --remote-read-src-addr=http://prometheus-stack-kube-prom-prometheus.prometheus.svc.cluster.local:9090
+            - --remote-read-filter-time-start=2026-09-06T10:26:14Z
+            - --remote-read-step-interval=day
+            - --vm-addr=http://victoria-metrics.prometheus.svc.cluster.local:8428
+          resources:
+            requests:
+              cpu: "200m"
+              memory: "512Mi"
+            limits:
+              cpu: "1000m"
+              memory: "1Gi"