diff --git a/monitoring/alertmanager/alertmanager.yml b/monitoring/alertmanager/alertmanager.yml index 841250b..ba592b7 100644 --- a/monitoring/alertmanager/alertmanager.yml +++ b/monitoring/alertmanager/alertmanager.yml @@ -5,25 +5,12 @@ route: group_wait: 1m group_interval: 5m repeat_interval: 4h - routes: - # VORUEBERGEHEND STUMM (gitops#47): Die CVE-Regeln feuern eine Alarm- - # Instanz pro CVE pro Image. Messung an 3 der 29 Images ergab 27 CRITICAL - # und 126 HIGH -> hochgerechnet ~260 CRITICAL / ~1200 HIGH. matrix-alerts.py - # schickt eine Nachricht pro Alarm; Synapse rate-limitet nach ~10, die - # Exception verhindert save_state(), der Receiver antwortet 502 und - # Alertmanager wiederholt die komplette Gruppe -> Dauerschleife. - # Regeln laufen bewusst weiter (Zahlen sichtbar in Prometheus/Grafana), - # nur die Zustellung haengt hier. Wieder scharf schalten: diese route raus, - # sobald der Receiver Sammelnachrichten kann bzw. die Regeln aggregieren. - - matchers: - - room="security" - receiver: "null" + # CVE-Alarme laufen seit gitops#51 aggregiert (eine Instanz pro Image statt + # pro CVE) und wieder ueber den Matrix-Receiver — Historie des Alarm-Sturms + # und der Stummschaltung: gitops#51 / AAR 2026-08-01. receivers: - name: matrix webhook_configs: - url: http://matrix-alerts:8080/alert send_resolved: true - - # Schwarzes Loch fuer stummgeschaltete Routen - bewusst ohne Konfiguration. - - name: "null" diff --git a/monitoring/alertmanager/matrix-alerts.py b/monitoring/alertmanager/matrix-alerts.py index 9f65aee..bb0c7fb 100644 --- a/monitoring/alertmanager/matrix-alerts.py +++ b/monitoring/alertmanager/matrix-alerts.py @@ -12,6 +12,7 @@ import html import json import os +import time import urllib.parse import urllib.request import uuid @@ -30,15 +31,18 @@ ROOM_MAP = {k[len("MATRIX_ROOM_"):].lower(): v try: with open(STATE_FILE) as f: - state = json.load(f) # fingerprint -> {"event_id":..., "text":...} + _persisted = json.load(f) + state = _persisted.get("alerts", _persisted) # alt: flaches Format + recent_resolved = _persisted.get("recent_resolved", []) except Exception: state = {} + recent_resolved = [] def save_state(): try: with open(STATE_FILE, "w") as f: - json.dump(state, f) + json.dump({"alerts": state, "recent_resolved": recent_resolved[-200:]}, f) except Exception as e: print(f"state save failed: {e}", flush=True) @@ -74,28 +78,38 @@ class Handler(BaseHTTPRequestHandler): if self.path != "/alert": self.send_response(404); self.end_headers(); return data = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0)))) - try: - for a in data.get("alerts", []): - name = a.get("labels", {}).get("alertname", "?") - summary = a.get("annotations", {}).get("summary", "") - fp = a.get("fingerprint", "") - room = ROOM_MAP.get(a.get("labels", {}).get("room", ""), ROOM_ID) + failed = 0 + for a in data.get("alerts", []): + name = a.get("labels", {}).get("alertname", "?") + summary = a.get("annotations", {}).get("summary", "") + fp = a.get("fingerprint", "") + room = ROOM_MAP.get(a.get("labels", {}).get("room", ""), ROOM_ID) + try: if a.get("status") == "firing": - text = f"\U0001F534 [firing] {name}: {summary}" - if fp in state: # re-notify desselben Alarms -> keine Doppelnachricht + if fp in state: # re-notify/Batch-Retry -> keine Doppelnachricht continue + text = f"\U0001F534 [firing] {name}: {summary}" event_id = send_text(text, room) if fp and event_id: state[fp] = {"event_id": event_id, "text": text, "room": room} else: + if fp in recent_resolved: # Batch-Retry -> Fallback nicht doppeln + continue known = state.pop(fp, None) if known: edit_resolved(known["event_id"], known["text"], known.get("room", ROOM_ID)) else: send_text(f"✅ [resolved] {name}: {summary}", room) - save_state() - except Exception as e: - self.log_message("matrix send failed: %s", e) + recent_resolved.append(fp) + save_state() # inkrementell: Teilfortschritt uebersteht Fehler/Retry + time.sleep(1) # Synapse-Ratelimit (rc_message) nicht reizen + except Exception as e: + failed += 1 + self.log_message("matrix send failed (%s): %s", name, e) + save_state() + time.sleep(2) + if failed: + # Alertmanager wiederholt den Batch; Dedup liefert nur den Rest nach self.send_response(502); self.end_headers(); return self.send_response(200); self.end_headers() diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml index d5ad81d..13dbc79 100644 --- a/monitoring/prometheus/alerts.yml +++ b/monitoring/prometheus/alerts.yml @@ -58,21 +58,26 @@ groups: # Alle Regeln routen per room-Label in den Security-Raum. - name: axion-cve rules: - - alert: TrivyCriticalVuln - expr: trivy_vuln_info{severity="CRITICAL"} == 1 + # Aggregiert pro Image statt pro CVE (gitops#51, Entscheidung sorb + # 2026-08-01): ~1000 Einzelalarme waeren Rauschen; die menschlich + # handhabbare Einheit ist "Image X hat N kritische CVEs". Die + # CVE-Details (Pflichtfelder) liegen im Grafana-Dashboard cve-overview, + # die trivy_vuln_info-Einzelserien bleiben dafuer erhalten. + - alert: TrivyCriticalVulns + expr: count by (target, target_type, host) (trivy_vuln_info{severity="CRITICAL"}) > 0 labels: severity: critical room: security annotations: - summary: "{{ $labels.cve }} (CRITICAL) in {{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}] — Paket {{ $labels.pkg }} {{ $labels.installed }}{{ if $labels.fixed_version }}, Fix: {{ $labels.fixed_version }}{{ else }}, noch kein Fix{{ end }} — https://nvd.nist.gov/vuln/detail/{{ $labels.cve }}" - - alert: TrivyHighVuln - expr: trivy_vuln_info{severity="HIGH"} == 1 + summary: "{{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}]: {{ $value }} CRITICAL-CVEs — Details: https://selendis.axion1337.de/d/cve-overview" + - alert: TrivyHighVulns + expr: count by (target, target_type, host) (trivy_vuln_info{severity="HIGH"}) > 0 for: 24h labels: severity: warning room: security annotations: - summary: "{{ $labels.cve }} (HIGH) in {{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}] — Paket {{ $labels.pkg }} {{ $labels.installed }}{{ if $labels.fixed_version }}, Fix: {{ $labels.fixed_version }}{{ else }}, noch kein Fix{{ end }} — https://nvd.nist.gov/vuln/detail/{{ $labels.cve }}" + summary: "{{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}]: {{ $value }} HIGH-CVEs (seit 24h offen) — Details: https://selendis.axion1337.de/d/cve-overview" - alert: TrivyScanStale expr: time() - trivy_last_scan_timestamp > 172800 labels: