From b6007c50fd69284637a96eb97eac90da917f8fe4 Mon Sep 17 00:00:00 2001 From: Thore Cimbal Date: Sat, 1 Aug 2026 13:42:07 +0200 Subject: [PATCH] monitoring: CVE-Pipeline v1 (gitops#47) - Scanner, Exporter, Regeln, Routing, Dashboard - cve-scan: Trivy-Loop ueber die 29 real deployten Images (Cluster-Inventur 2026-08-01 + Prod-Web-Image); 24h-Intervall, Fehler einzelner Images blockieren nicht - cve-exporter: Stdlib-Exporter mit first_seen-State (Zeitstrahl), Schema trivy_vuln_info/_count/_first_seen/_last_scan gemaess Pflichtfeldern - 3 Alertregeln (CRITICAL sofort, HIGH mit 24h-Daempfung, Scan-Frische) - promtool SUCCESS 9 rules; alle mit room=security - matrix-alerts: Label-basiertes Raum-Routing (MATRIX_ROOM_), Edits landen im richtigen Raum via State - Grafana-Dashboard cve-overview: Severity-Stats, CVE-Tabelle mit NVD-Link/Fix-Version/first-seen, Zeitstrahl, Verlauf UNGETESTET bis zum Deploy auf CFGMON (compose up -d). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PKhFj1S3UdD6xL2fbWPeYj --- monitoring/alertmanager/matrix-alerts.py | 27 +++-- monitoring/cve/cve-exporter.py | 98 +++++++++++++++++++ monitoring/cve/images.txt | 29 ++++++ monitoring/cve/scan-loop.sh | 23 +++++ monitoring/docker-compose.yml | 35 +++++++ .../dashboards/security/cve-overview.json | 64 ++++++++++++ monitoring/prometheus/alerts.yml | 29 ++++++ monitoring/prometheus/prometheus.yml | 5 + 8 files changed, 300 insertions(+), 10 deletions(-) create mode 100644 monitoring/cve/cve-exporter.py create mode 100644 monitoring/cve/images.txt create mode 100644 monitoring/cve/scan-loop.sh create mode 100644 monitoring/grafana/dashboards/security/cve-overview.json diff --git a/monitoring/alertmanager/matrix-alerts.py b/monitoring/alertmanager/matrix-alerts.py index aa3b301..9f65aee 100644 --- a/monitoring/alertmanager/matrix-alerts.py +++ b/monitoring/alertmanager/matrix-alerts.py @@ -21,6 +21,12 @@ HOMESERVER = os.environ["MATRIX_HOMESERVER"] ROOM_ID = os.environ["MATRIX_ROOM_ID"] TOKEN = os.environ["MATRIX_TOKEN"] STATE_FILE = os.environ.get("MATRIX_STATE_FILE", "/tmp/matrix-alerts-state.json") +# Raum-Routing (gitops#47): Alerts mit Label room="" landen im Raum aus +# MATRIX_ROOM_ (z.B. room="security" -> MATRIX_ROOM_SECURITY), alles +# andere im Default-Raum. Der Bot muss in jedem Zielraum Mitglied sein. +ROOM_MAP = {k[len("MATRIX_ROOM_"):].lower(): v + for k, v in os.environ.items() + if k.startswith("MATRIX_ROOM_") and k != "MATRIX_ROOM_ID" and v} try: with open(STATE_FILE) as f: @@ -37,8 +43,8 @@ def save_state(): print(f"state save failed: {e}", flush=True) -def matrix_put(content): - url = (f"{HOMESERVER}/_matrix/client/v3/rooms/{urllib.parse.quote(ROOM_ID)}" +def matrix_put(content, room=ROOM_ID): + url = (f"{HOMESERVER}/_matrix/client/v3/rooms/{urllib.parse.quote(room)}" f"/send/m.room.message/{uuid.uuid4()}") req = urllib.request.Request(url, data=json.dumps(content).encode(), method="PUT", headers={"Authorization": f"Bearer {TOKEN}", @@ -46,11 +52,11 @@ def matrix_put(content): return json.loads(urllib.request.urlopen(req, timeout=10).read()).get("event_id") -def send_text(text): - return matrix_put({"msgtype": "m.text", "body": text}) +def send_text(text, room=ROOM_ID): + return matrix_put({"msgtype": "m.text", "body": text}, room) -def edit_resolved(event_id, old_text): +def edit_resolved(event_id, old_text, room=ROOM_ID): # m.replace-Edit: Original wird in Element in-place ersetzt (durchgestrichen + Haken) new_body = f"✅ ~~{old_text}~~" new_html = f"✅ {html.escape(old_text)}" @@ -60,7 +66,7 @@ def edit_resolved(event_id, old_text): "m.new_content": {"msgtype": "m.text", "body": new_body, "format": "org.matrix.custom.html", "formatted_body": new_html}, "m.relates_to": {"rel_type": "m.replace", "event_id": event_id}, - }) + }, room) class Handler(BaseHTTPRequestHandler): @@ -73,19 +79,20 @@ class Handler(BaseHTTPRequestHandler): name = a.get("labels", {}).get("alertname", "?") summary = a.get("annotations", {}).get("summary", "") fp = a.get("fingerprint", "") + room = ROOM_MAP.get(a.get("labels", {}).get("room", ""), ROOM_ID) if a.get("status") == "firing": text = f"\U0001F534 [firing] {name}: {summary}" if fp in state: # re-notify desselben Alarms -> keine Doppelnachricht continue - event_id = send_text(text) + event_id = send_text(text, room) if fp and event_id: - state[fp] = {"event_id": event_id, "text": text} + state[fp] = {"event_id": event_id, "text": text, "room": room} else: known = state.pop(fp, None) if known: - edit_resolved(known["event_id"], known["text"]) + edit_resolved(known["event_id"], known["text"], known.get("room", ROOM_ID)) else: - send_text(f"✅ [resolved] {name}: {summary}") + send_text(f"✅ [resolved] {name}: {summary}", room) save_state() except Exception as e: self.log_message("matrix send failed: %s", e) diff --git a/monitoring/cve/cve-exporter.py b/monitoring/cve/cve-exporter.py new file mode 100644 index 0000000..c9a5364 --- /dev/null +++ b/monitoring/cve/cve-exporter.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# Trivy-JSON -> Prometheus-Metriken (gitops#47). Stdlib-only, Machart wie die +# uebrigen Monitoring-Helfer. Liest die Reports des cve-scan-Sidecars und +# serviert /metrics; merkt sich je (CVE, Target) den Erstfund (first_seen), +# damit der geforderte Zeitstrahl (erstmals gesehen / geschlossen) abbildbar +# ist - "geschlossen" = Serie verschwindet, resolved kommt via Alertmanager. +# +# Schema (Pflichtfelder-Vorgabe sorb: CVE-ID, Mitigation, Zeitstrahl, Ort, Typ): +# trivy_vuln_info{cve,severity,target,target_type,host,pkg,installed,fixed_version} 1 +# (nur HIGH/CRITICAL als Einzelserien - Kardinalitaet) +# trivy_vuln_count{target,target_type,host,severity} (alle Severities) +# trivy_vuln_first_seen_timestamp{cve,target} +# trivy_last_scan_timestamp{target,target_type,host} +import json +import os +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +RESULTS = os.environ.get("RESULTS_DIR", "/results") +STATE_FILE = os.environ.get("STATE_FILE", "/state/first-seen.json") +# Ort: Container-Images laufen (bis auf Weiteres) alle auf dem MATRIX-Host; +# Host-rootfs-Scans (Ausbaustufe) bringen ihren Hostnamen im Dateinamen mit. +DEFAULT_HOST = os.environ.get("DEFAULT_HOST", "matrix") + +try: + with open(STATE_FILE) as f: + first_seen = json.load(f) # "cve|target" -> unix-ts +except Exception: + first_seen = {} + + +def esc(v): + return str(v).replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ") + + +def collect(): + lines = [] + now = int(time.time()) + seen_keys = set() + for fn in sorted(os.listdir(RESULTS)): + if not fn.endswith(".json"): + continue + path = os.path.join(RESULTS, fn) + try: + rep = json.load(open(path)) + except Exception: + continue + target = rep.get("ArtifactName", fn[:-5]) + ttype = "host" if rep.get("ArtifactType") in ("filesystem", "rootfs") else "image" + host = fn.split("__host__")[1].split(".json")[0] if "__host__" in fn else DEFAULT_HOST + base = f'target="{esc(target)}",target_type="{ttype}",host="{esc(host)}"' + lines.append(f'trivy_last_scan_timestamp{{{base}}} {int(os.path.getmtime(path))}') + counts = {} + for res in rep.get("Results") or []: + for v in res.get("Vulnerabilities") or []: + sev = v.get("Severity", "UNKNOWN") + counts[sev] = counts.get(sev, 0) + 1 + if sev in ("HIGH", "CRITICAL"): + cve = v.get("VulnerabilityID", "?") + key = f"{cve}|{target}" + if key not in first_seen: + first_seen[key] = now + seen_keys.add(key) + lines.append( + 'trivy_vuln_info{cve="%s",severity="%s",%s,pkg="%s",installed="%s",fixed_version="%s"} 1' + % (esc(cve), sev, base, esc(v.get("PkgName", "?")), + esc(v.get("InstalledVersion", "?")), esc(v.get("FixedVersion", "")))) + lines.append(f'trivy_vuln_first_seen_timestamp{{cve="{esc(cve)}",target="{esc(target)}"}} {first_seen[key]}') + for sev, n in sorted(counts.items()): + lines.append(f'trivy_vuln_count{{{base},severity="{sev}"}} {n}') + # State kompakt halten: verschwundene Findings raus (= "geschlossen") + for k in list(first_seen): + if k not in seen_keys: + del first_seen[k] + try: + os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True) + with open(STATE_FILE, "w") as f: + json.dump(first_seen, f) + except Exception as e: + print(f"state save failed: {e}", flush=True) + return "\n".join(lines) + "\n" + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path != "/metrics": + self.send_response(404); self.end_headers(); return + body = collect().encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4") + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + pass + + +ThreadingHTTPServer(("0.0.0.0", 9101), Handler).serve_forever() diff --git a/monitoring/cve/images.txt b/monitoring/cve/images.txt new file mode 100644 index 0000000..2c246eb --- /dev/null +++ b/monitoring/cve/images.txt @@ -0,0 +1,29 @@ +clamav/clamav:1.5.3 +coturn/coturn:latest +docker.io/grafana/alloy:v1.7.5 +docker.io/library/haproxy:3.2-alpine +docker.io/library/postgres:17.9-bookworm +docker.io/livekit/livekit-server:v1.10.0 +docker.io/postgres:17-alpine +docker.io/prometheuscommunity/postgres-exporter:v0.18.1 +ghcr.io/element-hq/ess-helm/matrix-tools:0.7.3 +ghcr.io/element-hq/matrix-authentication-service:1.15.0 +ghcr.io/fluxcd/helm-controller:v1.5.3 +ghcr.io/fluxcd/kustomize-controller:v1.8.3 +ghcr.io/fluxcd/notification-controller:v1.8.3 +ghcr.io/fluxcd/source-controller:v1.8.2 +ghcr.io/goauthentik/server:2026.2.3 +gnuxie/draupnir:v3.1.0 +nginx:1.26-alpine +oci.element.io/element-admin:0.1.11 +oci.element.io/lk-jwt-service:0.4.2 +oci.element.io/synapse:v1.151.0-ess.1 +quay.io/jetstack/cert-manager-cainjector:v1.14.0 +quay.io/jetstack/cert-manager-controller:v1.14.0 +quay.io/jetstack/cert-manager-webhook:v1.14.0 +quay.io/prometheus-operator/prometheus-config-reloader:v0.81.0 +registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.15.0 +rohana.axion1337.de/sorb/axion-backup:v2 +rohana.axion1337.de/sorb/axion-secret-rotation:v1 +rohana.axion1337.de/sorb/clamav-http-scanner:v1.0.0 +rohana.axion1337.de/sorb/threadnet-web:v0.3.0 diff --git a/monitoring/cve/scan-loop.sh b/monitoring/cve/scan-loop.sh new file mode 100644 index 0000000..5751e4b --- /dev/null +++ b/monitoring/cve/scan-loop.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Trivy-Scan-Schleife (gitops#47): scannt alle 24h die Liste der real deployten +# Images (cve/images.txt, aus dem Cluster inventarisiert) und legt die +# JSON-Reports fuer den cve-exporter ab. Kein Abbruch bei Einzelfehlern - +# ein nicht mehr pullbares Image darf den Rest nicht verhindern (der Exporter +# alarmiert ueber trivy_last_scan_timestamp, wenn ein Report veraltet). +set -u +RESULTS=/results +IMAGES=/config/images.txt +INTERVAL="${SCAN_INTERVAL_SECONDS:-86400}" + +while true; do + while IFS= read -r img; do + case "$img" in ""|\#*) continue;; esac + safe=$(echo "$img" | tr '/:@' '___') + echo "scan: $img" + trivy image --scanners vuln --format json --output "$RESULTS/$safe.json.tmp" "$img" \ + && mv "$RESULTS/$safe.json.tmp" "$RESULTS/$safe.json" \ + || echo "FEHLER bei $img (Report bleibt auf altem Stand)" + done < "$IMAGES" + echo "runde fertig, schlafe ${INTERVAL}s" + sleep "$INTERVAL" +done diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index 310db01..a181464 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -45,6 +45,8 @@ services: - MATRIX_HOMESERVER=${MATRIX_ALERT_HOMESERVER} - MATRIX_ROOM_ID=${MATRIX_ALERT_ROOM_ID} - MATRIX_TOKEN=${MATRIX_ALERT_TOKEN} + # Raum-Routing (gitops#47): Alerts mit Label room=security -> Security-Raum + - MATRIX_ROOM_SECURITY=${MATRIX_RELEASE_ROOM_ID:-} # Ohne das liegt der State unter /tmp im Writable Layer: der ueberlebt # zwar ein "compose restart", aber kein "up -d", das den Container neu # baut -- also genau jeden Deploy. Dann verlieren offene Alarme ihre @@ -78,6 +80,36 @@ services: networks: - traefik + # CVE-Pipeline (gitops#47), Teil 1: Trivy scannt die real deployten Images + # (cve/images.txt - Inventur aus dem Cluster, bei Stack-Aenderungen nachziehen) + cve-scan: + image: aquasec/trivy:0.58.2 + container_name: cve-scan + restart: unless-stopped + entrypoint: ["/bin/sh", "/config/scan-loop.sh"] + volumes: + - ./cve:/config:ro + - cve_results:/results + - cve_trivy_cache:/root/.cache + networks: + - traefik + + # Teil 2: Reports -> Prometheus-Metriken (Schema + Pflichtfelder siehe gitops#47) + cve-exporter: + image: python:3.13-slim + container_name: cve-exporter + restart: unless-stopped + environment: + - RESULTS_DIR=/results + - STATE_FILE=/state/first-seen.json + volumes: + - ./cve/cve-exporter.py:/app/cve-exporter.py:ro + - cve_results:/results:ro + - cve_exporter_state:/state + command: python3 /app/cve-exporter.py + networks: + - traefik + loki: image: grafana/loki:3.7.1 container_name: loki @@ -166,6 +198,9 @@ volumes: alertmanager_data: matrix_alerts_data: release_watch_data: + cve_results: + cve_trivy_cache: + cve_exporter_state: grafana_data: loki_data: alloy_data: diff --git a/monitoring/grafana/dashboards/security/cve-overview.json b/monitoring/grafana/dashboards/security/cve-overview.json new file mode 100644 index 0000000..77bb8fb --- /dev/null +++ b/monitoring/grafana/dashboards/security/cve-overview.json @@ -0,0 +1,64 @@ +{ + "title": "CVE-Übersicht (Trivy)", + "uid": "cve-overview", + "tags": ["security", "cve", "gitops-47"], + "timezone": "browser", + "schemaVersion": 39, + "refresh": "5m", + "time": { "from": "now-30d", "to": "now" }, + "panels": [ + { + "type": "stat", "title": "Offene CRITICAL", "id": 1, + "gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, + "targets": [ { "expr": "sum(trivy_vuln_count{severity=\"CRITICAL\"}) or vector(0)", "instant": true, "refId": "A" } ] + }, + { + "type": "stat", "title": "Offene HIGH", "id": 2, + "gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] } }, "overrides": [] }, + "targets": [ { "expr": "sum(trivy_vuln_count{severity=\"HIGH\"}) or vector(0)", "instant": true, "refId": "A" } ] + }, + { + "type": "stat", "title": "MEDIUM/LOW (Summe)", "id": 3, + "gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 }, + "targets": [ { "expr": "sum(trivy_vuln_count{severity=~\"MEDIUM|LOW\"}) or vector(0)", "instant": true, "refId": "A" } ] + }, + { + "type": "stat", "title": "Ältester Scan (Stunden)", "id": 4, + "gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 }, + "fieldConfig": { "defaults": { "unit": "h", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 48 } ] } }, "overrides": [] }, + "targets": [ { "expr": "(time() - min(trivy_last_scan_timestamp)) / 3600", "instant": true, "refId": "A" } ] + }, + { + "type": "table", "title": "Offene CVEs (HIGH/CRITICAL) — CVE · Ort · Typ · Paket · Fix · seit", "id": 5, + "gridPos": { "x": 0, "y": 4, "w": 24, "h": 12 }, + "targets": [ + { "expr": "trivy_vuln_info", "instant": true, "format": "table", "refId": "A" }, + { "expr": "trivy_vuln_first_seen_timestamp * 1000", "instant": true, "format": "table", "refId": "B" } + ], + "transformations": [ + { "id": "merge", "options": {} }, + { "id": "organize", "options": { + "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "Value #A": true }, + "renameByName": { "cve": "CVE", "severity": "Severity", "host": "Ort (Host)", "target_type": "Typ", "target": "Target", "pkg": "Paket", "installed": "Installiert", "fixed_version": "Fix-Version", "Value #B": "Erstmals gesehen" } + } } + ], + "fieldConfig": { "defaults": {}, "overrides": [ + { "matcher": { "id": "byName", "options": "Erstmals gesehen" }, "properties": [ { "id": "unit", "value": "dateTimeAsIso" } ] }, + { "matcher": { "id": "byName", "options": "CVE" }, "properties": [ { "id": "links", "value": [ { "title": "NVD (Mitigation/Details)", "targetBlank": true, "url": "https://nvd.nist.gov/vuln/detail/${__value.text}" } ] } ] } + ] } + }, + { + "type": "state-timeline", "title": "Zeitstrahl: offene HIGH/CRITICAL je Target", "id": 6, + "gridPos": { "x": 0, "y": 16, "w": 24, "h": 8 }, + "targets": [ { "expr": "count by (target) (trivy_vuln_info)", "legendFormat": "{{target}}", "refId": "A" } ], + "fieldConfig": { "defaults": { "custom": { "fillOpacity": 70 } }, "overrides": [] } + }, + { + "type": "timeseries", "title": "Funde je Severity über Zeit", "id": 7, + "gridPos": { "x": 0, "y": 24, "w": 24, "h": 8 }, + "targets": [ { "expr": "sum by (severity) (trivy_vuln_count)", "legendFormat": "{{severity}}", "refId": "A" } ] + } + ] +} diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml index 7624cad..d5ad81d 100644 --- a/monitoring/prometheus/alerts.yml +++ b/monitoring/prometheus/alerts.yml @@ -51,3 +51,32 @@ groups: severity: critical annotations: summary: "{{ $labels.instance }}: <10% Platz auf / frei" + + # CVE-Pipeline (gitops#47): Funde aus den Trivy-Scans. Pflichtfelder je Alarm: + # CVE-ID, Mitigation (Fix-Version + NVD-Link), Ort (host), Typ (target_type); + # der Zeitstrahl kommt aus trivy_vuln_first_seen_timestamp + resolved-Edit. + # Alle Regeln routen per room-Label in den Security-Raum. + - name: axion-cve + rules: + - alert: TrivyCriticalVuln + expr: trivy_vuln_info{severity="CRITICAL"} == 1 + labels: + severity: critical + room: security + annotations: + summary: "{{ $labels.cve }} (CRITICAL) in {{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}] — Paket {{ $labels.pkg }} {{ $labels.installed }}{{ if $labels.fixed_version }}, Fix: {{ $labels.fixed_version }}{{ else }}, noch kein Fix{{ end }} — https://nvd.nist.gov/vuln/detail/{{ $labels.cve }}" + - alert: TrivyHighVuln + expr: trivy_vuln_info{severity="HIGH"} == 1 + for: 24h + labels: + severity: warning + room: security + annotations: + summary: "{{ $labels.cve }} (HIGH) in {{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}] — Paket {{ $labels.pkg }} {{ $labels.installed }}{{ if $labels.fixed_version }}, Fix: {{ $labels.fixed_version }}{{ else }}, noch kein Fix{{ end }} — https://nvd.nist.gov/vuln/detail/{{ $labels.cve }}" + - alert: TrivyScanStale + expr: time() - trivy_last_scan_timestamp > 172800 + labels: + severity: warning + room: security + annotations: + summary: "CVE-Scan fuer {{ $labels.target }} ist aelter als 2 Tage — Scanner pruefen (ein still gestorbener Scanner macht blind)" diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index d98de03..2d5f72c 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -47,3 +47,8 @@ scrape_configs: - job_name: "gameserver_cadvisor" static_configs: - targets: ["157.90.155.206:8080"] + + # CVE-Exporter (gitops#47) + - job_name: "cve_exporter" + static_configs: + - targets: ["cve-exporter:9101"]