monitoring: Alerting vorbereitet - Alertmanager, 6 Alert-Regeln, Matrix-Receiver (gitops#32)

Regeln decken die real erlebten Fehlerklassen ab: TargetDown (coturn-Klasse),
KubePodRestartLoop (36k-Restarts-Klasse), OOM-Kills, RAM-/Swap-/Disk-Druck.
Zustellung in den wartung-Raum ueber einen minimalen Stdlib-Webhook-Receiver
(gleiche Machart wie maintenance-notify, gitops Issue #24); Bot-Token kommt
beim Deploy per .env (Vorlage in .env.example).

UNGETESTET/DEPLOY-PENDING: promtool/amtool-Lint auf dem Mac an haengendem
Docker-Hub-Pull gescheitert - vor dem Deploy auf CFGMON ausfuehren (Images
liegen dort bereits):
  docker run --rm -v $PWD/monitoring/prometheus:/cfg:ro --entrypoint promtool prom/prometheus:v3.3.1 check rules /cfg/alerts.yml
  docker run --rm -v $PWD/monitoring/alertmanager:/cfg:ro --entrypoint amtool prom/alertmanager:v0.28.1 check-config /cfg/alertmanager.yml

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

monitoring/.env.example

@@ -2,3 +2,10 @@
 # Werte in Single-Quotes, damit Sonderzeichen shell-safe sind.
 GRAFANA_ADMIN_USER='admin'
 GRAFANA_ADMIN_PASSWORD='changeme'
+
+# Alerting (gitops#32) - Matrix-Zustellung in den wartung-Raum.
+# Token erzeugen: kubectl exec -n matrix deploy/matrix-stack-matrix-authentication-service -- \
+#   mas-cli manage issue-compatibility-token maintenance-notify
+MATRIX_ALERT_HOMESERVER=https://matrix.axion1337.chat
+MATRIX_ALERT_ROOM_ID=!lmZaajvVboTPxQxXzv:axion1337.chat
+MATRIX_ALERT_TOKEN=changeme

monitoring/alertmanager/alertmanager.yml

@@ -0,0 +1,13 @@
+# Alertmanager (gitops#32): alles an den Matrix-Receiver (wartung-Raum).
+route:
+  receiver: matrix
+  group_by: [alertname, instance]
+  group_wait: 1m
+  group_interval: 5m
+  repeat_interval: 4h
+
+receivers:
+  - name: matrix
+    webhook_configs:
+      - url: http://matrix-alerts:8080/alert
+        send_resolved: true

monitoring/alertmanager/matrix-alerts.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# Minimaler Alertmanager-Webhook -> Matrix-Raum (gitops#32). Gleiche Machart wie
+# maintenance-notify (Issue #24 im gitops-Repo): purer Stdlib-HTTP-Server, Bot-Token
+# aus der Umgebung, Nachricht per Client-Server-API. Kein Framework, kein State.
+import json, os, urllib.request, uuid
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+HOMESERVER = os.environ["MATRIX_HOMESERVER"]
+ROOM_ID = os.environ["MATRIX_ROOM_ID"]
+TOKEN = os.environ["MATRIX_TOKEN"]
+
+def send_matrix(text):
+    url = f"{HOMESERVER}/_matrix/client/v3/rooms/{urllib.parse.quote(ROOM_ID)}/send/m.room.message/{uuid.uuid4()}"
+    body = json.dumps({"msgtype": "m.text", "body": text}).encode()
+    req = urllib.request.Request(url, data=body, method="PUT", headers={
+        "Authorization": f"Bearer {TOKEN}", "Content-Type": "application/json"})
+    urllib.request.urlopen(req, timeout=10).read()
+
+class Handler(BaseHTTPRequestHandler):
+    def do_POST(self):
+        if self.path != "/alert":
+            self.send_response(404); self.end_headers(); return
+        data = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))))
+        lines = []
+        for a in data.get("alerts", []):
+            icon = "\U0001F534" if a.get("status") == "firing" else "✅"
+            name = a.get("labels", {}).get("alertname", "?")
+            summary = a.get("annotations", {}).get("summary", "")
+            lines.append(f"{icon} [{a.get('status')}] {name}: {summary}")
+        if lines:
+            try:
+                send_matrix("\n".join(lines))
+            except Exception as e:
+                self.log_message("matrix send failed: %s", e)
+                self.send_response(502); self.end_headers(); return
+        self.send_response(200); self.end_headers()
+
+    def log_message(self, fmt, *args):
+        print(fmt % args, flush=True)
+
+import urllib.parse
+ThreadingHTTPServer(("0.0.0.0", 8080), Handler).serve_forever()

monitoring/docker-compose.yml

@@ -5,6 +5,7 @@ services:
     restart: unless-stopped
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
       - prometheus_data:/prometheus
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
@@ -20,6 +21,36 @@ services:
     networks:
       - traefik
 
+  alertmanager:
+    image: prom/alertmanager:v0.28.1
+    container_name: alertmanager
+    restart: unless-stopped
+    volumes:
+      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+      - alertmanager_data:/alertmanager
+    command:
+      - '--config.file=/etc/alertmanager/alertmanager.yml'
+      - '--storage.path=/alertmanager'
+    # bewusst kein oeffentlicher Port - nur Prometheus/Receiver im traefik-Netz
+    networks:
+      - traefik
+
+  # Alertmanager-Webhook -> Matrix (wartung-Raum), gleiche Machart wie
+  # maintenance-notify (gitops Issue #24). Secrets kommen aus .env.
+  matrix-alerts:
+    image: python:3.13-slim
+    container_name: matrix-alerts
+    restart: unless-stopped
+    environment:
+      - MATRIX_HOMESERVER=${MATRIX_ALERT_HOMESERVER}
+      - MATRIX_ROOM_ID=${MATRIX_ALERT_ROOM_ID}
+      - MATRIX_TOKEN=${MATRIX_ALERT_TOKEN}
+    volumes:
+      - ./alertmanager/matrix-alerts.py:/app/matrix-alerts.py:ro
+    command: python3 /app/matrix-alerts.py
+    networks:
+      - traefik
+
   loki:
     image: grafana/loki:3.7.1
     container_name: loki
@@ -105,6 +136,7 @@ services:
 
 volumes:
   prometheus_data:
+  alertmanager_data:
   grafana_data:
   loki_data:
   alloy_data:

monitoring/prometheus/alerts.yml

@@ -0,0 +1,53 @@
+# Alert-Regeln (gitops#32). Bewusst wenige, dafuer die real erlebten Fehlerklassen:
+# - TargetDown: "Exporter/Dienst tot und keiner merkt es" (coturn-Klasse)
+# - KubePodRestartLoop: Crashloops im k3s-Cluster (36k-coturn-Restarts-Klasse)
+# - HostOom/HostLowMemory/HostLowDisk/HostHighSwap: Ressourcendruck der Hosts
+groups:
+  - name: axion-basics
+    rules:
+      - alert: TargetDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Target {{ $labels.job }}/{{ $labels.instance }} ist seit 5m nicht erreichbar"
+
+      - alert: KubePodRestartLoop
+        expr: increase(kube_pod_container_status_restarts_total[1h]) > 3
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} restartet wiederholt ({{ $value | printf \"%.0f\" }}x in 1h)"
+
+      - alert: HostOomKill
+        expr: increase(node_vmstat_oom_kill[15m]) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: "OOM-Kill auf {{ $labels.instance }}"
+
+      - alert: HostLowMemory
+        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.08
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.instance }}: <8% RAM verfuegbar seit 15m"
+
+      - alert: HostHighSwap
+        expr: (node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes > 0.5
+        for: 30m
+        labels:
+          severity: warning
+        annotations:
+          summary: "{{ $labels.instance }}: >50% Swap in Nutzung seit 30m"
+
+      - alert: HostLowDisk
+        expr: node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} < 0.10
+        for: 15m
+        labels:
+          severity: critical
+        annotations:
+          summary: "{{ $labels.instance }}: <10% Platz auf / frei"

monitoring/prometheus/prometheus.yml

@@ -1,6 +1,14 @@
 global:
   scrape_interval: 15s
 
+rule_files:
+  - /etc/prometheus/alerts.yml
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+
 # Zusaetzlich zu diesen Scrape-Targets kommen Metriken per Remote-Write rein
 # (k3s-Cluster: flux, kube_state_metrics; Matrix-Server: synapse).
 scrape_configs: