From 9594ec80ecffce8c50719989cba80052200afcb8 Mon Sep 17 00:00:00 2001 From: Thore Cimbal Date: Fri, 31 Jul 2026 18:30:40 +0200 Subject: [PATCH] monitoring: Alerting vorbereitet - Alertmanager, 6 Alert-Regeln, Matrix-Receiver (gitops#32) Regeln decken die real erlebten Fehlerklassen ab: TargetDown (coturn-Klasse), KubePodRestartLoop (36k-Restarts-Klasse), OOM-Kills, RAM-/Swap-/Disk-Druck. Zustellung in den wartung-Raum ueber einen minimalen Stdlib-Webhook-Receiver (gleiche Machart wie maintenance-notify, gitops Issue #24); Bot-Token kommt beim Deploy per .env (Vorlage in .env.example). UNGETESTET/DEPLOY-PENDING: promtool/amtool-Lint auf dem Mac an haengendem Docker-Hub-Pull gescheitert - vor dem Deploy auf CFGMON ausfuehren (Images liegen dort bereits): docker run --rm -v $PWD/monitoring/prometheus:/cfg:ro --entrypoint promtool prom/prometheus:v3.3.1 check rules /cfg/alerts.yml docker run --rm -v $PWD/monitoring/alertmanager:/cfg:ro --entrypoint amtool prom/alertmanager:v0.28.1 check-config /cfg/alertmanager.yml Co-Authored-By: Claude Fable 5 --- monitoring/.env.example | 7 ++++ monitoring/alertmanager/alertmanager.yml | 13 ++++++ monitoring/alertmanager/matrix-alerts.py | 42 +++++++++++++++++++ monitoring/docker-compose.yml | 32 ++++++++++++++ monitoring/prometheus/alerts.yml | 53 ++++++++++++++++++++++++ monitoring/prometheus/prometheus.yml | 8 ++++ 6 files changed, 155 insertions(+) create mode 100644 monitoring/alertmanager/alertmanager.yml create mode 100644 monitoring/alertmanager/matrix-alerts.py create mode 100644 monitoring/prometheus/alerts.yml diff --git a/monitoring/.env.example b/monitoring/.env.example index d011fc4..536a56f 100644 --- a/monitoring/.env.example +++ b/monitoring/.env.example @@ -2,3 +2,10 @@ # Werte in Single-Quotes, damit Sonderzeichen shell-safe sind. GRAFANA_ADMIN_USER='admin' GRAFANA_ADMIN_PASSWORD='changeme' + +# Alerting (gitops#32) - Matrix-Zustellung in den wartung-Raum. +# Token erzeugen: kubectl exec -n matrix deploy/matrix-stack-matrix-authentication-service -- \ +# mas-cli manage issue-compatibility-token maintenance-notify +MATRIX_ALERT_HOMESERVER=https://matrix.axion1337.chat +MATRIX_ALERT_ROOM_ID=!lmZaajvVboTPxQxXzv:axion1337.chat +MATRIX_ALERT_TOKEN=changeme diff --git a/monitoring/alertmanager/alertmanager.yml b/monitoring/alertmanager/alertmanager.yml new file mode 100644 index 0000000..29b9510 --- /dev/null +++ b/monitoring/alertmanager/alertmanager.yml @@ -0,0 +1,13 @@ +# Alertmanager (gitops#32): alles an den Matrix-Receiver (wartung-Raum). +route: + receiver: matrix + group_by: [alertname, instance] + group_wait: 1m + group_interval: 5m + repeat_interval: 4h + +receivers: + - name: matrix + webhook_configs: + - url: http://matrix-alerts:8080/alert + send_resolved: true diff --git a/monitoring/alertmanager/matrix-alerts.py b/monitoring/alertmanager/matrix-alerts.py new file mode 100644 index 0000000..56851a8 --- /dev/null +++ b/monitoring/alertmanager/matrix-alerts.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Minimaler Alertmanager-Webhook -> Matrix-Raum (gitops#32). Gleiche Machart wie +# maintenance-notify (Issue #24 im gitops-Repo): purer Stdlib-HTTP-Server, Bot-Token +# aus der Umgebung, Nachricht per Client-Server-API. Kein Framework, kein State. +import json, os, urllib.request, uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +HOMESERVER = os.environ["MATRIX_HOMESERVER"] +ROOM_ID = os.environ["MATRIX_ROOM_ID"] +TOKEN = os.environ["MATRIX_TOKEN"] + +def send_matrix(text): + url = f"{HOMESERVER}/_matrix/client/v3/rooms/{urllib.parse.quote(ROOM_ID)}/send/m.room.message/{uuid.uuid4()}" + body = json.dumps({"msgtype": "m.text", "body": text}).encode() + req = urllib.request.Request(url, data=body, method="PUT", headers={ + "Authorization": f"Bearer {TOKEN}", "Content-Type": "application/json"}) + urllib.request.urlopen(req, timeout=10).read() + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + if self.path != "/alert": + self.send_response(404); self.end_headers(); return + data = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0)))) + lines = [] + for a in data.get("alerts", []): + icon = "\U0001F534" if a.get("status") == "firing" else "✅" + name = a.get("labels", {}).get("alertname", "?") + summary = a.get("annotations", {}).get("summary", "") + lines.append(f"{icon} [{a.get('status')}] {name}: {summary}") + if lines: + try: + send_matrix("\n".join(lines)) + except Exception as e: + self.log_message("matrix send failed: %s", e) + self.send_response(502); self.end_headers(); return + self.send_response(200); self.end_headers() + + def log_message(self, fmt, *args): + print(fmt % args, flush=True) + +import urllib.parse +ThreadingHTTPServer(("0.0.0.0", 8080), Handler).serve_forever() diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index 92bfca1..fbddb8a 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -5,6 +5,7 @@ services: restart: unless-stopped volumes: - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro - prometheus_data:/prometheus command: - '--config.file=/etc/prometheus/prometheus.yml' @@ -20,6 +21,36 @@ services: networks: - traefik + alertmanager: + image: prom/alertmanager:v0.28.1 + container_name: alertmanager + restart: unless-stopped + volumes: + - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - alertmanager_data:/alertmanager + command: + - '--config.file=/etc/alertmanager/alertmanager.yml' + - '--storage.path=/alertmanager' + # bewusst kein oeffentlicher Port - nur Prometheus/Receiver im traefik-Netz + networks: + - traefik + + # Alertmanager-Webhook -> Matrix (wartung-Raum), gleiche Machart wie + # maintenance-notify (gitops Issue #24). Secrets kommen aus .env. + matrix-alerts: + image: python:3.13-slim + container_name: matrix-alerts + restart: unless-stopped + environment: + - MATRIX_HOMESERVER=${MATRIX_ALERT_HOMESERVER} + - MATRIX_ROOM_ID=${MATRIX_ALERT_ROOM_ID} + - MATRIX_TOKEN=${MATRIX_ALERT_TOKEN} + volumes: + - ./alertmanager/matrix-alerts.py:/app/matrix-alerts.py:ro + command: python3 /app/matrix-alerts.py + networks: + - traefik + loki: image: grafana/loki:3.7.1 container_name: loki @@ -105,6 +136,7 @@ services: volumes: prometheus_data: + alertmanager_data: grafana_data: loki_data: alloy_data: diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml new file mode 100644 index 0000000..7624cad --- /dev/null +++ b/monitoring/prometheus/alerts.yml @@ -0,0 +1,53 @@ +# Alert-Regeln (gitops#32). Bewusst wenige, dafuer die real erlebten Fehlerklassen: +# - TargetDown: "Exporter/Dienst tot und keiner merkt es" (coturn-Klasse) +# - KubePodRestartLoop: Crashloops im k3s-Cluster (36k-coturn-Restarts-Klasse) +# - HostOom/HostLowMemory/HostLowDisk/HostHighSwap: Ressourcendruck der Hosts +groups: + - name: axion-basics + rules: + - alert: TargetDown + expr: up == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Target {{ $labels.job }}/{{ $labels.instance }} ist seit 5m nicht erreichbar" + + - alert: KubePodRestartLoop + expr: increase(kube_pod_container_status_restarts_total[1h]) > 3 + for: 10m + labels: + severity: warning + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} restartet wiederholt ({{ $value | printf \"%.0f\" }}x in 1h)" + + - alert: HostOomKill + expr: increase(node_vmstat_oom_kill[15m]) > 0 + labels: + severity: warning + annotations: + summary: "OOM-Kill auf {{ $labels.instance }}" + + - alert: HostLowMemory + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.08 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $labels.instance }}: <8% RAM verfuegbar seit 15m" + + - alert: HostHighSwap + expr: (node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes > 0.5 + for: 30m + labels: + severity: warning + annotations: + summary: "{{ $labels.instance }}: >50% Swap in Nutzung seit 30m" + + - alert: HostLowDisk + expr: node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} < 0.10 + for: 15m + labels: + severity: critical + annotations: + summary: "{{ $labels.instance }}: <10% Platz auf / frei" diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index 3f1ab8c..d98de03 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -1,6 +1,14 @@ global: scrape_interval: 15s +rule_files: + - /etc/prometheus/alerts.yml + +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] + # Zusaetzlich zu diesen Scrape-Targets kommen Metriken per Remote-Write rein # (k3s-Cluster: flux, kube_state_metrics; Matrix-Server: synapse). scrape_configs: