monitoring: CVE-Alarme aggregiert pro Image + Receiver-Robustheit (gitops#51)
Entscheidung sorb 2026-08-01 (Option 1 aus #51): Alarme als count by (target, severity) statt pro CVE (~58 Serien statt ~1200), CVE-Details bleiben im Dashboard (trivy_vuln_info unveraendert). Receiver: inkrementelles save_state nach jedem Alarm, 1s-Sende-Drossel (Synapse rc_message), recent_resolved-Dedup gegen doppelte Fallback-Haken bei Batch-Retries, Teilfehler -> 502 liefert nur den Rest nach. Stumm-Route + Null-Receiver entfernt - Zustellung wieder scharf. promtool/amtool/py_compile gruen. UNGETESTET bis Deploy (Uebergabe-Issue). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PKhFj1S3UdD6xL2fbWPeYj
This commit is contained in:
Thore Cimbal
co-authored by
Claude Fable 5
Claude Fable 5
parent
0bd77e28d8
commit
ff87cb25fb
@@ -5,25 +5,12 @@ route:
|
||||
group_wait: 1m
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
# VORUEBERGEHEND STUMM (gitops#47): Die CVE-Regeln feuern eine Alarm-
|
||||
# Instanz pro CVE pro Image. Messung an 3 der 29 Images ergab 27 CRITICAL
|
||||
# und 126 HIGH -> hochgerechnet ~260 CRITICAL / ~1200 HIGH. matrix-alerts.py
|
||||
# schickt eine Nachricht pro Alarm; Synapse rate-limitet nach ~10, die
|
||||
# Exception verhindert save_state(), der Receiver antwortet 502 und
|
||||
# Alertmanager wiederholt die komplette Gruppe -> Dauerschleife.
|
||||
# Regeln laufen bewusst weiter (Zahlen sichtbar in Prometheus/Grafana),
|
||||
# nur die Zustellung haengt hier. Wieder scharf schalten: diese route raus,
|
||||
# sobald der Receiver Sammelnachrichten kann bzw. die Regeln aggregieren.
|
||||
- matchers:
|
||||
- room="security"
|
||||
receiver: "null"
|
||||
# CVE-Alarme laufen seit gitops#51 aggregiert (eine Instanz pro Image statt
|
||||
# pro CVE) und wieder ueber den Matrix-Receiver — Historie des Alarm-Sturms
|
||||
# und der Stummschaltung: gitops#51 / AAR 2026-08-01.
|
||||
|
||||
receivers:
|
||||
- name: matrix
|
||||
webhook_configs:
|
||||
- url: http://matrix-alerts:8080/alert
|
||||
send_resolved: true
|
||||
|
||||
# Schwarzes Loch fuer stummgeschaltete Routen - bewusst ohne Konfiguration.
|
||||
- name: "null"
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
@@ -30,15 +31,18 @@ ROOM_MAP = {k[len("MATRIX_ROOM_"):].lower(): v
|
||||
|
||||
try:
|
||||
with open(STATE_FILE) as f:
|
||||
state = json.load(f) # fingerprint -> {"event_id":..., "text":...}
|
||||
_persisted = json.load(f)
|
||||
state = _persisted.get("alerts", _persisted) # alt: flaches Format
|
||||
recent_resolved = _persisted.get("recent_resolved", [])
|
||||
except Exception:
|
||||
state = {}
|
||||
recent_resolved = []
|
||||
|
||||
|
||||
def save_state():
|
||||
try:
|
||||
with open(STATE_FILE, "w") as f:
|
||||
json.dump(state, f)
|
||||
json.dump({"alerts": state, "recent_resolved": recent_resolved[-200:]}, f)
|
||||
except Exception as e:
|
||||
print(f"state save failed: {e}", flush=True)
|
||||
|
||||
@@ -74,28 +78,38 @@ class Handler(BaseHTTPRequestHandler):
|
||||
if self.path != "/alert":
|
||||
self.send_response(404); self.end_headers(); return
|
||||
data = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))))
|
||||
try:
|
||||
for a in data.get("alerts", []):
|
||||
name = a.get("labels", {}).get("alertname", "?")
|
||||
summary = a.get("annotations", {}).get("summary", "")
|
||||
fp = a.get("fingerprint", "")
|
||||
room = ROOM_MAP.get(a.get("labels", {}).get("room", ""), ROOM_ID)
|
||||
failed = 0
|
||||
for a in data.get("alerts", []):
|
||||
name = a.get("labels", {}).get("alertname", "?")
|
||||
summary = a.get("annotations", {}).get("summary", "")
|
||||
fp = a.get("fingerprint", "")
|
||||
room = ROOM_MAP.get(a.get("labels", {}).get("room", ""), ROOM_ID)
|
||||
try:
|
||||
if a.get("status") == "firing":
|
||||
text = f"\U0001F534 [firing] {name}: {summary}"
|
||||
if fp in state: # re-notify desselben Alarms -> keine Doppelnachricht
|
||||
if fp in state: # re-notify/Batch-Retry -> keine Doppelnachricht
|
||||
continue
|
||||
text = f"\U0001F534 [firing] {name}: {summary}"
|
||||
event_id = send_text(text, room)
|
||||
if fp and event_id:
|
||||
state[fp] = {"event_id": event_id, "text": text, "room": room}
|
||||
else:
|
||||
if fp in recent_resolved: # Batch-Retry -> Fallback nicht doppeln
|
||||
continue
|
||||
known = state.pop(fp, None)
|
||||
if known:
|
||||
edit_resolved(known["event_id"], known["text"], known.get("room", ROOM_ID))
|
||||
else:
|
||||
send_text(f"✅ [resolved] {name}: {summary}", room)
|
||||
save_state()
|
||||
except Exception as e:
|
||||
self.log_message("matrix send failed: %s", e)
|
||||
recent_resolved.append(fp)
|
||||
save_state() # inkrementell: Teilfortschritt uebersteht Fehler/Retry
|
||||
time.sleep(1) # Synapse-Ratelimit (rc_message) nicht reizen
|
||||
except Exception as e:
|
||||
failed += 1
|
||||
self.log_message("matrix send failed (%s): %s", name, e)
|
||||
save_state()
|
||||
time.sleep(2)
|
||||
if failed:
|
||||
# Alertmanager wiederholt den Batch; Dedup liefert nur den Rest nach
|
||||
self.send_response(502); self.end_headers(); return
|
||||
self.send_response(200); self.end_headers()
|
||||
|
||||
|
||||
@@ -58,21 +58,26 @@ groups:
|
||||
# Alle Regeln routen per room-Label in den Security-Raum.
|
||||
- name: axion-cve
|
||||
rules:
|
||||
- alert: TrivyCriticalVuln
|
||||
expr: trivy_vuln_info{severity="CRITICAL"} == 1
|
||||
# Aggregiert pro Image statt pro CVE (gitops#51, Entscheidung sorb
|
||||
# 2026-08-01): ~1000 Einzelalarme waeren Rauschen; die menschlich
|
||||
# handhabbare Einheit ist "Image X hat N kritische CVEs". Die
|
||||
# CVE-Details (Pflichtfelder) liegen im Grafana-Dashboard cve-overview,
|
||||
# die trivy_vuln_info-Einzelserien bleiben dafuer erhalten.
|
||||
- alert: TrivyCriticalVulns
|
||||
expr: count by (target, target_type, host) (trivy_vuln_info{severity="CRITICAL"}) > 0
|
||||
labels:
|
||||
severity: critical
|
||||
room: security
|
||||
annotations:
|
||||
summary: "{{ $labels.cve }} (CRITICAL) in {{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}] — Paket {{ $labels.pkg }} {{ $labels.installed }}{{ if $labels.fixed_version }}, Fix: {{ $labels.fixed_version }}{{ else }}, noch kein Fix{{ end }} — https://nvd.nist.gov/vuln/detail/{{ $labels.cve }}"
|
||||
- alert: TrivyHighVuln
|
||||
expr: trivy_vuln_info{severity="HIGH"} == 1
|
||||
summary: "{{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}]: {{ $value }} CRITICAL-CVEs — Details: https://selendis.axion1337.de/d/cve-overview"
|
||||
- alert: TrivyHighVulns
|
||||
expr: count by (target, target_type, host) (trivy_vuln_info{severity="HIGH"}) > 0
|
||||
for: 24h
|
||||
labels:
|
||||
severity: warning
|
||||
room: security
|
||||
annotations:
|
||||
summary: "{{ $labels.cve }} (HIGH) in {{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}] — Paket {{ $labels.pkg }} {{ $labels.installed }}{{ if $labels.fixed_version }}, Fix: {{ $labels.fixed_version }}{{ else }}, noch kein Fix{{ end }} — https://nvd.nist.gov/vuln/detail/{{ $labels.cve }}"
|
||||
summary: "{{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}]: {{ $value }} HIGH-CVEs (seit 24h offen) — Details: https://selendis.axion1337.de/d/cve-overview"
|
||||
- alert: TrivyScanStale
|
||||
expr: time() - trivy_last_scan_timestamp > 172800
|
||||
labels:
|
||||
|
||||
Reference in New Issue
Block a user