monitoring: CVE-Pipeline v1 (gitops#47) - Scanner, Exporter, Regeln, Routing, Dashboard

- cve-scan: Trivy-Loop ueber die 29 real deployten Images (Cluster-Inventur
  2026-08-01 + Prod-Web-Image); 24h-Intervall, Fehler einzelner Images
  blockieren nicht
- cve-exporter: Stdlib-Exporter mit first_seen-State (Zeitstrahl), Schema
  trivy_vuln_info/_count/_first_seen/_last_scan gemaess Pflichtfeldern
- 3 Alertregeln (CRITICAL sofort, HIGH mit 24h-Daempfung, Scan-Frische) -
  promtool SUCCESS 9 rules; alle mit room=security
- matrix-alerts: Label-basiertes Raum-Routing (MATRIX_ROOM_<NAME>), Edits
  landen im richtigen Raum via State
- Grafana-Dashboard cve-overview: Severity-Stats, CVE-Tabelle mit
  NVD-Link/Fix-Version/first-seen, Zeitstrahl, Verlauf
UNGETESTET bis zum Deploy auf CFGMON (compose up -d).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PKhFj1S3UdD6xL2fbWPeYj

monitoring/alertmanager/matrix-alerts.py

@@ -21,6 +21,12 @@ HOMESERVER = os.environ["MATRIX_HOMESERVER"]
 ROOM_ID = os.environ["MATRIX_ROOM_ID"]
 TOKEN = os.environ["MATRIX_TOKEN"]
 STATE_FILE = os.environ.get("MATRIX_STATE_FILE", "/tmp/matrix-alerts-state.json")
+# Raum-Routing (gitops#47): Alerts mit Label room="<name>" landen im Raum aus
+# MATRIX_ROOM_<NAME> (z.B. room="security" -> MATRIX_ROOM_SECURITY), alles
+# andere im Default-Raum. Der Bot muss in jedem Zielraum Mitglied sein.
+ROOM_MAP = {k[len("MATRIX_ROOM_"):].lower(): v
+            for k, v in os.environ.items()
+            if k.startswith("MATRIX_ROOM_") and k != "MATRIX_ROOM_ID" and v}
 
 try:
     with open(STATE_FILE) as f:
@@ -37,8 +43,8 @@ def save_state():
         print(f"state save failed: {e}", flush=True)
 
 
-def matrix_put(content):
+def matrix_put(content, room=ROOM_ID):
-    url = (f"{HOMESERVER}/_matrix/client/v3/rooms/{urllib.parse.quote(ROOM_ID)}"
+    url = (f"{HOMESERVER}/_matrix/client/v3/rooms/{urllib.parse.quote(room)}"
            f"/send/m.room.message/{uuid.uuid4()}")
     req = urllib.request.Request(url, data=json.dumps(content).encode(), method="PUT",
                                  headers={"Authorization": f"Bearer {TOKEN}",
@@ -46,11 +52,11 @@ def matrix_put(content):
     return json.loads(urllib.request.urlopen(req, timeout=10).read()).get("event_id")
 
 
-def send_text(text):
+def send_text(text, room=ROOM_ID):
-    return matrix_put({"msgtype": "m.text", "body": text})
+    return matrix_put({"msgtype": "m.text", "body": text}, room)
 
 
-def edit_resolved(event_id, old_text):
+def edit_resolved(event_id, old_text, room=ROOM_ID):
     # m.replace-Edit: Original wird in Element in-place ersetzt (durchgestrichen + Haken)
     new_body = f"✅ ~~{old_text}~~"
     new_html = f"✅ <del>{html.escape(old_text)}</del>"
@@ -60,7 +66,7 @@ def edit_resolved(event_id, old_text):
         "m.new_content": {"msgtype": "m.text", "body": new_body,
                           "format": "org.matrix.custom.html", "formatted_body": new_html},
         "m.relates_to": {"rel_type": "m.replace", "event_id": event_id},
-    })
+    }, room)
 
 
 class Handler(BaseHTTPRequestHandler):
@@ -73,19 +79,20 @@ class Handler(BaseHTTPRequestHandler):
                 name = a.get("labels", {}).get("alertname", "?")
                 summary = a.get("annotations", {}).get("summary", "")
                 fp = a.get("fingerprint", "")
+                room = ROOM_MAP.get(a.get("labels", {}).get("room", ""), ROOM_ID)
                 if a.get("status") == "firing":
                     text = f"\U0001F534 [firing] {name}: {summary}"
                     if fp in state:  # re-notify desselben Alarms -> keine Doppelnachricht
                         continue
-                    event_id = send_text(text)
+                    event_id = send_text(text, room)
                     if fp and event_id:
-                        state[fp] = {"event_id": event_id, "text": text}
+                        state[fp] = {"event_id": event_id, "text": text, "room": room}
                 else:
                     known = state.pop(fp, None)
                     if known:
-                        edit_resolved(known["event_id"], known["text"])
+                        edit_resolved(known["event_id"], known["text"], known.get("room", ROOM_ID))
                     else:
-                        send_text(f"✅ [resolved] {name}: {summary}")
+                        send_text(f"✅ [resolved] {name}: {summary}", room)
             save_state()
         except Exception as e:
             self.log_message("matrix send failed: %s", e)

monitoring/cve/cve-exporter.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+# Trivy-JSON -> Prometheus-Metriken (gitops#47). Stdlib-only, Machart wie die
+# uebrigen Monitoring-Helfer. Liest die Reports des cve-scan-Sidecars und
+# serviert /metrics; merkt sich je (CVE, Target) den Erstfund (first_seen),
+# damit der geforderte Zeitstrahl (erstmals gesehen / geschlossen) abbildbar
+# ist - "geschlossen" = Serie verschwindet, resolved kommt via Alertmanager.
+#
+# Schema (Pflichtfelder-Vorgabe sorb: CVE-ID, Mitigation, Zeitstrahl, Ort, Typ):
+#   trivy_vuln_info{cve,severity,target,target_type,host,pkg,installed,fixed_version} 1
+#       (nur HIGH/CRITICAL als Einzelserien - Kardinalitaet)
+#   trivy_vuln_count{target,target_type,host,severity} <n>   (alle Severities)
+#   trivy_vuln_first_seen_timestamp{cve,target} <unix>
+#   trivy_last_scan_timestamp{target,target_type,host} <unix>
+import json
+import os
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+RESULTS = os.environ.get("RESULTS_DIR", "/results")
+STATE_FILE = os.environ.get("STATE_FILE", "/state/first-seen.json")
+# Ort: Container-Images laufen (bis auf Weiteres) alle auf dem MATRIX-Host;
+# Host-rootfs-Scans (Ausbaustufe) bringen ihren Hostnamen im Dateinamen mit.
+DEFAULT_HOST = os.environ.get("DEFAULT_HOST", "matrix")
+
+try:
+    with open(STATE_FILE) as f:
+        first_seen = json.load(f)  # "cve|target" -> unix-ts
+except Exception:
+    first_seen = {}
+
+
+def esc(v):
+    return str(v).replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ")
+
+
+def collect():
+    lines = []
+    now = int(time.time())
+    seen_keys = set()
+    for fn in sorted(os.listdir(RESULTS)):
+        if not fn.endswith(".json"):
+            continue
+        path = os.path.join(RESULTS, fn)
+        try:
+            rep = json.load(open(path))
+        except Exception:
+            continue
+        target = rep.get("ArtifactName", fn[:-5])
+        ttype = "host" if rep.get("ArtifactType") in ("filesystem", "rootfs") else "image"
+        host = fn.split("__host__")[1].split(".json")[0] if "__host__" in fn else DEFAULT_HOST
+        base = f'target="{esc(target)}",target_type="{ttype}",host="{esc(host)}"'
+        lines.append(f'trivy_last_scan_timestamp{{{base}}} {int(os.path.getmtime(path))}')
+        counts = {}
+        for res in rep.get("Results") or []:
+            for v in res.get("Vulnerabilities") or []:
+                sev = v.get("Severity", "UNKNOWN")
+                counts[sev] = counts.get(sev, 0) + 1
+                if sev in ("HIGH", "CRITICAL"):
+                    cve = v.get("VulnerabilityID", "?")
+                    key = f"{cve}|{target}"
+                    if key not in first_seen:
+                        first_seen[key] = now
+                    seen_keys.add(key)
+                    lines.append(
+                        'trivy_vuln_info{cve="%s",severity="%s",%s,pkg="%s",installed="%s",fixed_version="%s"} 1'
+                        % (esc(cve), sev, base, esc(v.get("PkgName", "?")),
+                           esc(v.get("InstalledVersion", "?")), esc(v.get("FixedVersion", ""))))
+                    lines.append(f'trivy_vuln_first_seen_timestamp{{cve="{esc(cve)}",target="{esc(target)}"}} {first_seen[key]}')
+        for sev, n in sorted(counts.items()):
+            lines.append(f'trivy_vuln_count{{{base},severity="{sev}"}} {n}')
+    # State kompakt halten: verschwundene Findings raus (= "geschlossen")
+    for k in list(first_seen):
+        if k not in seen_keys:
+            del first_seen[k]
+    try:
+        os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True)
+        with open(STATE_FILE, "w") as f:
+            json.dump(first_seen, f)
+    except Exception as e:
+        print(f"state save failed: {e}", flush=True)
+    return "\n".join(lines) + "\n"
+
+
+class Handler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path != "/metrics":
+            self.send_response(404); self.end_headers(); return
+        body = collect().encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4")
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, fmt, *args):
+        pass
+
+
+ThreadingHTTPServer(("0.0.0.0", 9101), Handler).serve_forever()

monitoring/cve/images.txt

@@ -0,0 +1,29 @@
+clamav/clamav:1.5.3
+coturn/coturn:latest
+docker.io/grafana/alloy:v1.7.5
+docker.io/library/haproxy:3.2-alpine
+docker.io/library/postgres:17.9-bookworm
+docker.io/livekit/livekit-server:v1.10.0
+docker.io/postgres:17-alpine
+docker.io/prometheuscommunity/postgres-exporter:v0.18.1
+ghcr.io/element-hq/ess-helm/matrix-tools:0.7.3
+ghcr.io/element-hq/matrix-authentication-service:1.15.0
+ghcr.io/fluxcd/helm-controller:v1.5.3
+ghcr.io/fluxcd/kustomize-controller:v1.8.3
+ghcr.io/fluxcd/notification-controller:v1.8.3
+ghcr.io/fluxcd/source-controller:v1.8.2
+ghcr.io/goauthentik/server:2026.2.3
+gnuxie/draupnir:v3.1.0
+nginx:1.26-alpine
+oci.element.io/element-admin:0.1.11
+oci.element.io/lk-jwt-service:0.4.2
+oci.element.io/synapse:v1.151.0-ess.1
+quay.io/jetstack/cert-manager-cainjector:v1.14.0
+quay.io/jetstack/cert-manager-controller:v1.14.0
+quay.io/jetstack/cert-manager-webhook:v1.14.0
+quay.io/prometheus-operator/prometheus-config-reloader:v0.81.0
+registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.15.0
+rohana.axion1337.de/sorb/axion-backup:v2
+rohana.axion1337.de/sorb/axion-secret-rotation:v1
+rohana.axion1337.de/sorb/clamav-http-scanner:v1.0.0
+rohana.axion1337.de/sorb/threadnet-web:v0.3.0

monitoring/cve/scan-loop.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+# Trivy-Scan-Schleife (gitops#47): scannt alle 24h die Liste der real deployten
+# Images (cve/images.txt, aus dem Cluster inventarisiert) und legt die
+# JSON-Reports fuer den cve-exporter ab. Kein Abbruch bei Einzelfehlern -
+# ein nicht mehr pullbares Image darf den Rest nicht verhindern (der Exporter
+# alarmiert ueber trivy_last_scan_timestamp, wenn ein Report veraltet).
+set -u
+RESULTS=/results
+IMAGES=/config/images.txt
+INTERVAL="${SCAN_INTERVAL_SECONDS:-86400}"
+
+while true; do
+  while IFS= read -r img; do
+    case "$img" in ""|\#*) continue;; esac
+    safe=$(echo "$img" | tr '/:@' '___')
+    echo "scan: $img"
+    trivy image --scanners vuln --format json --output "$RESULTS/$safe.json.tmp" "$img" \
+      && mv "$RESULTS/$safe.json.tmp" "$RESULTS/$safe.json" \
+      || echo "FEHLER bei $img (Report bleibt auf altem Stand)"
+  done < "$IMAGES"
+  echo "runde fertig, schlafe ${INTERVAL}s"
+  sleep "$INTERVAL"
+done

monitoring/docker-compose.yml

@@ -45,6 +45,8 @@ services:
       - MATRIX_HOMESERVER=${MATRIX_ALERT_HOMESERVER}
       - MATRIX_ROOM_ID=${MATRIX_ALERT_ROOM_ID}
       - MATRIX_TOKEN=${MATRIX_ALERT_TOKEN}
+      # Raum-Routing (gitops#47): Alerts mit Label room=security -> Security-Raum
+      - MATRIX_ROOM_SECURITY=${MATRIX_RELEASE_ROOM_ID:-}
       # Ohne das liegt der State unter /tmp im Writable Layer: der ueberlebt
       # zwar ein "compose restart", aber kein "up -d", das den Container neu
       # baut -- also genau jeden Deploy. Dann verlieren offene Alarme ihre
@@ -78,6 +80,36 @@ services:
     networks:
       - traefik
 
+  # CVE-Pipeline (gitops#47), Teil 1: Trivy scannt die real deployten Images
+  # (cve/images.txt - Inventur aus dem Cluster, bei Stack-Aenderungen nachziehen)
+  cve-scan:
+    image: aquasec/trivy:0.58.2
+    container_name: cve-scan
+    restart: unless-stopped
+    entrypoint: ["/bin/sh", "/config/scan-loop.sh"]
+    volumes:
+      - ./cve:/config:ro
+      - cve_results:/results
+      - cve_trivy_cache:/root/.cache
+    networks:
+      - traefik
+
+  # Teil 2: Reports -> Prometheus-Metriken (Schema + Pflichtfelder siehe gitops#47)
+  cve-exporter:
+    image: python:3.13-slim
+    container_name: cve-exporter
+    restart: unless-stopped
+    environment:
+      - RESULTS_DIR=/results
+      - STATE_FILE=/state/first-seen.json
+    volumes:
+      - ./cve/cve-exporter.py:/app/cve-exporter.py:ro
+      - cve_results:/results:ro
+      - cve_exporter_state:/state
+    command: python3 /app/cve-exporter.py
+    networks:
+      - traefik
+
   loki:
     image: grafana/loki:3.7.1
     container_name: loki
@@ -166,6 +198,9 @@ volumes:
   alertmanager_data:
   matrix_alerts_data:
   release_watch_data:
+  cve_results:
+  cve_trivy_cache:
+  cve_exporter_state:
   grafana_data:
   loki_data:
   alloy_data:

monitoring/grafana/dashboards/security/cve-overview.json

@@ -0,0 +1,64 @@
+{
+  "title": "CVE-Übersicht (Trivy)",
+  "uid": "cve-overview",
+  "tags": ["security", "cve", "gitops-47"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "refresh": "5m",
+  "time": { "from": "now-30d", "to": "now" },
+  "panels": [
+    {
+      "type": "stat", "title": "Offene CRITICAL", "id": 1,
+      "gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 },
+      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] },
+      "targets": [ { "expr": "sum(trivy_vuln_count{severity=\"CRITICAL\"}) or vector(0)", "instant": true, "refId": "A" } ]
+    },
+    {
+      "type": "stat", "title": "Offene HIGH", "id": 2,
+      "gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 },
+      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] } }, "overrides": [] },
+      "targets": [ { "expr": "sum(trivy_vuln_count{severity=\"HIGH\"}) or vector(0)", "instant": true, "refId": "A" } ]
+    },
+    {
+      "type": "stat", "title": "MEDIUM/LOW (Summe)", "id": 3,
+      "gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 },
+      "targets": [ { "expr": "sum(trivy_vuln_count{severity=~\"MEDIUM|LOW\"}) or vector(0)", "instant": true, "refId": "A" } ]
+    },
+    {
+      "type": "stat", "title": "Ältester Scan (Stunden)", "id": 4,
+      "gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 },
+      "fieldConfig": { "defaults": { "unit": "h", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 48 } ] } }, "overrides": [] },
+      "targets": [ { "expr": "(time() - min(trivy_last_scan_timestamp)) / 3600", "instant": true, "refId": "A" } ]
+    },
+    {
+      "type": "table", "title": "Offene CVEs (HIGH/CRITICAL) — CVE · Ort · Typ · Paket · Fix · seit", "id": 5,
+      "gridPos": { "x": 0, "y": 4, "w": 24, "h": 12 },
+      "targets": [
+        { "expr": "trivy_vuln_info", "instant": true, "format": "table", "refId": "A" },
+        { "expr": "trivy_vuln_first_seen_timestamp * 1000", "instant": true, "format": "table", "refId": "B" }
+      ],
+      "transformations": [
+        { "id": "merge", "options": {} },
+        { "id": "organize", "options": {
+            "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "Value #A": true },
+            "renameByName": { "cve": "CVE", "severity": "Severity", "host": "Ort (Host)", "target_type": "Typ", "target": "Target", "pkg": "Paket", "installed": "Installiert", "fixed_version": "Fix-Version", "Value #B": "Erstmals gesehen" }
+        } }
+      ],
+      "fieldConfig": { "defaults": {}, "overrides": [
+        { "matcher": { "id": "byName", "options": "Erstmals gesehen" }, "properties": [ { "id": "unit", "value": "dateTimeAsIso" } ] },
+        { "matcher": { "id": "byName", "options": "CVE" }, "properties": [ { "id": "links", "value": [ { "title": "NVD (Mitigation/Details)", "targetBlank": true, "url": "https://nvd.nist.gov/vuln/detail/${__value.text}" } ] } ] }
+      ] }
+    },
+    {
+      "type": "state-timeline", "title": "Zeitstrahl: offene HIGH/CRITICAL je Target", "id": 6,
+      "gridPos": { "x": 0, "y": 16, "w": 24, "h": 8 },
+      "targets": [ { "expr": "count by (target) (trivy_vuln_info)", "legendFormat": "{{target}}", "refId": "A" } ],
+      "fieldConfig": { "defaults": { "custom": { "fillOpacity": 70 } }, "overrides": [] }
+    },
+    {
+      "type": "timeseries", "title": "Funde je Severity über Zeit", "id": 7,
+      "gridPos": { "x": 0, "y": 24, "w": 24, "h": 8 },
+      "targets": [ { "expr": "sum by (severity) (trivy_vuln_count)", "legendFormat": "{{severity}}", "refId": "A" } ]
+    }
+  ]
+}

monitoring/prometheus/alerts.yml

@@ -51,3 +51,32 @@ groups:
           severity: critical
         annotations:
           summary: "{{ $labels.instance }}: <10% Platz auf / frei"
+
+  # CVE-Pipeline (gitops#47): Funde aus den Trivy-Scans. Pflichtfelder je Alarm:
+  # CVE-ID, Mitigation (Fix-Version + NVD-Link), Ort (host), Typ (target_type);
+  # der Zeitstrahl kommt aus trivy_vuln_first_seen_timestamp + resolved-Edit.
+  # Alle Regeln routen per room-Label in den Security-Raum.
+  - name: axion-cve
+    rules:
+      - alert: TrivyCriticalVuln
+        expr: trivy_vuln_info{severity="CRITICAL"} == 1
+        labels:
+          severity: critical
+          room: security
+        annotations:
+          summary: "{{ $labels.cve }} (CRITICAL) in {{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}] — Paket {{ $labels.pkg }} {{ $labels.installed }}{{ if $labels.fixed_version }}, Fix: {{ $labels.fixed_version }}{{ else }}, noch kein Fix{{ end }} — https://nvd.nist.gov/vuln/detail/{{ $labels.cve }}"
+      - alert: TrivyHighVuln
+        expr: trivy_vuln_info{severity="HIGH"} == 1
+        for: 24h
+        labels:
+          severity: warning
+          room: security
+        annotations:
+          summary: "{{ $labels.cve }} (HIGH) in {{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}] — Paket {{ $labels.pkg }} {{ $labels.installed }}{{ if $labels.fixed_version }}, Fix: {{ $labels.fixed_version }}{{ else }}, noch kein Fix{{ end }} — https://nvd.nist.gov/vuln/detail/{{ $labels.cve }}"
+      - alert: TrivyScanStale
+        expr: time() - trivy_last_scan_timestamp > 172800
+        labels:
+          severity: warning
+          room: security
+        annotations:
+          summary: "CVE-Scan fuer {{ $labels.target }} ist aelter als 2 Tage — Scanner pruefen (ein still gestorbener Scanner macht blind)"

monitoring/prometheus/prometheus.yml

@@ -47,3 +47,8 @@ scrape_configs:
   - job_name: "gameserver_cadvisor"
     static_configs:
       - targets: ["157.90.155.206:8080"]
+
+  # CVE-Exporter (gitops#47)
+  - job_name: "cve_exporter"
+    static_configs:
+      - targets: ["cve-exporter:9101"]