monitoring: CVE-Pipeline v1 (gitops#47) - Scanner, Exporter, Regeln, Routing, Dashboard

- cve-scan: Trivy-Loop ueber die 29 real deployten Images (Cluster-Inventur
  2026-08-01 + Prod-Web-Image); 24h-Intervall, Fehler einzelner Images
  blockieren nicht
- cve-exporter: Stdlib-Exporter mit first_seen-State (Zeitstrahl), Schema
  trivy_vuln_info/_count/_first_seen/_last_scan gemaess Pflichtfeldern
- 3 Alertregeln (CRITICAL sofort, HIGH mit 24h-Daempfung, Scan-Frische) -
  promtool SUCCESS 9 rules; alle mit room=security
- matrix-alerts: Label-basiertes Raum-Routing (MATRIX_ROOM_<NAME>), Edits
  landen im richtigen Raum via State
- Grafana-Dashboard cve-overview: Severity-Stats, CVE-Tabelle mit
  NVD-Link/Fix-Version/first-seen, Zeitstrahl, Verlauf
UNGETESTET bis zum Deploy auf CFGMON (compose up -d).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PKhFj1S3UdD6xL2fbWPeYj
This commit is contained in:
Thore Cimbal
2026-08-01 13:42:07 +02:00
co-authored by Claude Fable 5
parent f45e01219b
commit b6007c50fd
8 changed files with 300 additions and 10 deletions
+17 -10
View File
@@ -21,6 +21,12 @@ HOMESERVER = os.environ["MATRIX_HOMESERVER"]
ROOM_ID = os.environ["MATRIX_ROOM_ID"]
TOKEN = os.environ["MATRIX_TOKEN"]
STATE_FILE = os.environ.get("MATRIX_STATE_FILE", "/tmp/matrix-alerts-state.json")
# Raum-Routing (gitops#47): Alerts mit Label room="<name>" landen im Raum aus
# MATRIX_ROOM_<NAME> (z.B. room="security" -> MATRIX_ROOM_SECURITY), alles
# andere im Default-Raum. Der Bot muss in jedem Zielraum Mitglied sein.
ROOM_MAP = {k[len("MATRIX_ROOM_"):].lower(): v
for k, v in os.environ.items()
if k.startswith("MATRIX_ROOM_") and k != "MATRIX_ROOM_ID" and v}
try:
with open(STATE_FILE) as f:
@@ -37,8 +43,8 @@ def save_state():
print(f"state save failed: {e}", flush=True)
def matrix_put(content):
url = (f"{HOMESERVER}/_matrix/client/v3/rooms/{urllib.parse.quote(ROOM_ID)}"
def matrix_put(content, room=ROOM_ID):
url = (f"{HOMESERVER}/_matrix/client/v3/rooms/{urllib.parse.quote(room)}"
f"/send/m.room.message/{uuid.uuid4()}")
req = urllib.request.Request(url, data=json.dumps(content).encode(), method="PUT",
headers={"Authorization": f"Bearer {TOKEN}",
@@ -46,11 +52,11 @@ def matrix_put(content):
return json.loads(urllib.request.urlopen(req, timeout=10).read()).get("event_id")
def send_text(text):
return matrix_put({"msgtype": "m.text", "body": text})
def send_text(text, room=ROOM_ID):
return matrix_put({"msgtype": "m.text", "body": text}, room)
def edit_resolved(event_id, old_text):
def edit_resolved(event_id, old_text, room=ROOM_ID):
# m.replace-Edit: Original wird in Element in-place ersetzt (durchgestrichen + Haken)
new_body = f"✅ ~~{old_text}~~"
new_html = f"✅ <del>{html.escape(old_text)}</del>"
@@ -60,7 +66,7 @@ def edit_resolved(event_id, old_text):
"m.new_content": {"msgtype": "m.text", "body": new_body,
"format": "org.matrix.custom.html", "formatted_body": new_html},
"m.relates_to": {"rel_type": "m.replace", "event_id": event_id},
})
}, room)
class Handler(BaseHTTPRequestHandler):
@@ -73,19 +79,20 @@ class Handler(BaseHTTPRequestHandler):
name = a.get("labels", {}).get("alertname", "?")
summary = a.get("annotations", {}).get("summary", "")
fp = a.get("fingerprint", "")
room = ROOM_MAP.get(a.get("labels", {}).get("room", ""), ROOM_ID)
if a.get("status") == "firing":
text = f"\U0001F534 [firing] {name}: {summary}"
if fp in state: # re-notify desselben Alarms -> keine Doppelnachricht
continue
event_id = send_text(text)
event_id = send_text(text, room)
if fp and event_id:
state[fp] = {"event_id": event_id, "text": text}
state[fp] = {"event_id": event_id, "text": text, "room": room}
else:
known = state.pop(fp, None)
if known:
edit_resolved(known["event_id"], known["text"])
edit_resolved(known["event_id"], known["text"], known.get("room", ROOM_ID))
else:
send_text(f"✅ [resolved] {name}: {summary}")
send_text(f"✅ [resolved] {name}: {summary}", room)
save_state()
except Exception as e:
self.log_message("matrix send failed: %s", e)
+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
# Trivy-JSON -> Prometheus-Metriken (gitops#47). Stdlib-only, Machart wie die
# uebrigen Monitoring-Helfer. Liest die Reports des cve-scan-Sidecars und
# serviert /metrics; merkt sich je (CVE, Target) den Erstfund (first_seen),
# damit der geforderte Zeitstrahl (erstmals gesehen / geschlossen) abbildbar
# ist - "geschlossen" = Serie verschwindet, resolved kommt via Alertmanager.
#
# Schema (Pflichtfelder-Vorgabe sorb: CVE-ID, Mitigation, Zeitstrahl, Ort, Typ):
# trivy_vuln_info{cve,severity,target,target_type,host,pkg,installed,fixed_version} 1
# (nur HIGH/CRITICAL als Einzelserien - Kardinalitaet)
# trivy_vuln_count{target,target_type,host,severity} <n> (alle Severities)
# trivy_vuln_first_seen_timestamp{cve,target} <unix>
# trivy_last_scan_timestamp{target,target_type,host} <unix>
import json
import os
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
RESULTS = os.environ.get("RESULTS_DIR", "/results")
STATE_FILE = os.environ.get("STATE_FILE", "/state/first-seen.json")
# Ort: Container-Images laufen (bis auf Weiteres) alle auf dem MATRIX-Host;
# Host-rootfs-Scans (Ausbaustufe) bringen ihren Hostnamen im Dateinamen mit.
DEFAULT_HOST = os.environ.get("DEFAULT_HOST", "matrix")
try:
with open(STATE_FILE) as f:
first_seen = json.load(f) # "cve|target" -> unix-ts
except Exception:
first_seen = {}
def esc(v):
return str(v).replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ")
def collect():
lines = []
now = int(time.time())
seen_keys = set()
for fn in sorted(os.listdir(RESULTS)):
if not fn.endswith(".json"):
continue
path = os.path.join(RESULTS, fn)
try:
rep = json.load(open(path))
except Exception:
continue
target = rep.get("ArtifactName", fn[:-5])
ttype = "host" if rep.get("ArtifactType") in ("filesystem", "rootfs") else "image"
host = fn.split("__host__")[1].split(".json")[0] if "__host__" in fn else DEFAULT_HOST
base = f'target="{esc(target)}",target_type="{ttype}",host="{esc(host)}"'
lines.append(f'trivy_last_scan_timestamp{{{base}}} {int(os.path.getmtime(path))}')
counts = {}
for res in rep.get("Results") or []:
for v in res.get("Vulnerabilities") or []:
sev = v.get("Severity", "UNKNOWN")
counts[sev] = counts.get(sev, 0) + 1
if sev in ("HIGH", "CRITICAL"):
cve = v.get("VulnerabilityID", "?")
key = f"{cve}|{target}"
if key not in first_seen:
first_seen[key] = now
seen_keys.add(key)
lines.append(
'trivy_vuln_info{cve="%s",severity="%s",%s,pkg="%s",installed="%s",fixed_version="%s"} 1'
% (esc(cve), sev, base, esc(v.get("PkgName", "?")),
esc(v.get("InstalledVersion", "?")), esc(v.get("FixedVersion", ""))))
lines.append(f'trivy_vuln_first_seen_timestamp{{cve="{esc(cve)}",target="{esc(target)}"}} {first_seen[key]}')
for sev, n in sorted(counts.items()):
lines.append(f'trivy_vuln_count{{{base},severity="{sev}"}} {n}')
# State kompakt halten: verschwundene Findings raus (= "geschlossen")
for k in list(first_seen):
if k not in seen_keys:
del first_seen[k]
try:
os.makedirs(os.path.dirname(STATE_FILE), exist_ok=True)
with open(STATE_FILE, "w") as f:
json.dump(first_seen, f)
except Exception as e:
print(f"state save failed: {e}", flush=True)
return "\n".join(lines) + "\n"
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path != "/metrics":
self.send_response(404); self.end_headers(); return
body = collect().encode()
self.send_response(200)
self.send_header("Content-Type", "text/plain; version=0.0.4")
self.end_headers()
self.wfile.write(body)
def log_message(self, fmt, *args):
pass
ThreadingHTTPServer(("0.0.0.0", 9101), Handler).serve_forever()
+29
View File
@@ -0,0 +1,29 @@
clamav/clamav:1.5.3
coturn/coturn:latest
docker.io/grafana/alloy:v1.7.5
docker.io/library/haproxy:3.2-alpine
docker.io/library/postgres:17.9-bookworm
docker.io/livekit/livekit-server:v1.10.0
docker.io/postgres:17-alpine
docker.io/prometheuscommunity/postgres-exporter:v0.18.1
ghcr.io/element-hq/ess-helm/matrix-tools:0.7.3
ghcr.io/element-hq/matrix-authentication-service:1.15.0
ghcr.io/fluxcd/helm-controller:v1.5.3
ghcr.io/fluxcd/kustomize-controller:v1.8.3
ghcr.io/fluxcd/notification-controller:v1.8.3
ghcr.io/fluxcd/source-controller:v1.8.2
ghcr.io/goauthentik/server:2026.2.3
gnuxie/draupnir:v3.1.0
nginx:1.26-alpine
oci.element.io/element-admin:0.1.11
oci.element.io/lk-jwt-service:0.4.2
oci.element.io/synapse:v1.151.0-ess.1
quay.io/jetstack/cert-manager-cainjector:v1.14.0
quay.io/jetstack/cert-manager-controller:v1.14.0
quay.io/jetstack/cert-manager-webhook:v1.14.0
quay.io/prometheus-operator/prometheus-config-reloader:v0.81.0
registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.15.0
rohana.axion1337.de/sorb/axion-backup:v2
rohana.axion1337.de/sorb/axion-secret-rotation:v1
rohana.axion1337.de/sorb/clamav-http-scanner:v1.0.0
rohana.axion1337.de/sorb/threadnet-web:v0.3.0
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
# Trivy-Scan-Schleife (gitops#47): scannt alle 24h die Liste der real deployten
# Images (cve/images.txt, aus dem Cluster inventarisiert) und legt die
# JSON-Reports fuer den cve-exporter ab. Kein Abbruch bei Einzelfehlern -
# ein nicht mehr pullbares Image darf den Rest nicht verhindern (der Exporter
# alarmiert ueber trivy_last_scan_timestamp, wenn ein Report veraltet).
set -u
RESULTS=/results
IMAGES=/config/images.txt
INTERVAL="${SCAN_INTERVAL_SECONDS:-86400}"
while true; do
while IFS= read -r img; do
case "$img" in ""|\#*) continue;; esac
safe=$(echo "$img" | tr '/:@' '___')
echo "scan: $img"
trivy image --scanners vuln --format json --output "$RESULTS/$safe.json.tmp" "$img" \
&& mv "$RESULTS/$safe.json.tmp" "$RESULTS/$safe.json" \
|| echo "FEHLER bei $img (Report bleibt auf altem Stand)"
done < "$IMAGES"
echo "runde fertig, schlafe ${INTERVAL}s"
sleep "$INTERVAL"
done
+35
View File
@@ -45,6 +45,8 @@ services:
- MATRIX_HOMESERVER=${MATRIX_ALERT_HOMESERVER}
- MATRIX_ROOM_ID=${MATRIX_ALERT_ROOM_ID}
- MATRIX_TOKEN=${MATRIX_ALERT_TOKEN}
# Raum-Routing (gitops#47): Alerts mit Label room=security -> Security-Raum
- MATRIX_ROOM_SECURITY=${MATRIX_RELEASE_ROOM_ID:-}
# Ohne das liegt der State unter /tmp im Writable Layer: der ueberlebt
# zwar ein "compose restart", aber kein "up -d", das den Container neu
# baut -- also genau jeden Deploy. Dann verlieren offene Alarme ihre
@@ -78,6 +80,36 @@ services:
networks:
- traefik
# CVE-Pipeline (gitops#47), Teil 1: Trivy scannt die real deployten Images
# (cve/images.txt - Inventur aus dem Cluster, bei Stack-Aenderungen nachziehen)
cve-scan:
image: aquasec/trivy:0.58.2
container_name: cve-scan
restart: unless-stopped
entrypoint: ["/bin/sh", "/config/scan-loop.sh"]
volumes:
- ./cve:/config:ro
- cve_results:/results
- cve_trivy_cache:/root/.cache
networks:
- traefik
# Teil 2: Reports -> Prometheus-Metriken (Schema + Pflichtfelder siehe gitops#47)
cve-exporter:
image: python:3.13-slim
container_name: cve-exporter
restart: unless-stopped
environment:
- RESULTS_DIR=/results
- STATE_FILE=/state/first-seen.json
volumes:
- ./cve/cve-exporter.py:/app/cve-exporter.py:ro
- cve_results:/results:ro
- cve_exporter_state:/state
command: python3 /app/cve-exporter.py
networks:
- traefik
loki:
image: grafana/loki:3.7.1
container_name: loki
@@ -166,6 +198,9 @@ volumes:
alertmanager_data:
matrix_alerts_data:
release_watch_data:
cve_results:
cve_trivy_cache:
cve_exporter_state:
grafana_data:
loki_data:
alloy_data:
@@ -0,0 +1,64 @@
{
"title": "CVE-Übersicht (Trivy)",
"uid": "cve-overview",
"tags": ["security", "cve", "gitops-47"],
"timezone": "browser",
"schemaVersion": 39,
"refresh": "5m",
"time": { "from": "now-30d", "to": "now" },
"panels": [
{
"type": "stat", "title": "Offene CRITICAL", "id": 1,
"gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 },
"fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] },
"targets": [ { "expr": "sum(trivy_vuln_count{severity=\"CRITICAL\"}) or vector(0)", "instant": true, "refId": "A" } ]
},
{
"type": "stat", "title": "Offene HIGH", "id": 2,
"gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 },
"fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] } }, "overrides": [] },
"targets": [ { "expr": "sum(trivy_vuln_count{severity=\"HIGH\"}) or vector(0)", "instant": true, "refId": "A" } ]
},
{
"type": "stat", "title": "MEDIUM/LOW (Summe)", "id": 3,
"gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 },
"targets": [ { "expr": "sum(trivy_vuln_count{severity=~\"MEDIUM|LOW\"}) or vector(0)", "instant": true, "refId": "A" } ]
},
{
"type": "stat", "title": "Ältester Scan (Stunden)", "id": 4,
"gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 },
"fieldConfig": { "defaults": { "unit": "h", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 48 } ] } }, "overrides": [] },
"targets": [ { "expr": "(time() - min(trivy_last_scan_timestamp)) / 3600", "instant": true, "refId": "A" } ]
},
{
"type": "table", "title": "Offene CVEs (HIGH/CRITICAL) — CVE · Ort · Typ · Paket · Fix · seit", "id": 5,
"gridPos": { "x": 0, "y": 4, "w": 24, "h": 12 },
"targets": [
{ "expr": "trivy_vuln_info", "instant": true, "format": "table", "refId": "A" },
{ "expr": "trivy_vuln_first_seen_timestamp * 1000", "instant": true, "format": "table", "refId": "B" }
],
"transformations": [
{ "id": "merge", "options": {} },
{ "id": "organize", "options": {
"excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "Value #A": true },
"renameByName": { "cve": "CVE", "severity": "Severity", "host": "Ort (Host)", "target_type": "Typ", "target": "Target", "pkg": "Paket", "installed": "Installiert", "fixed_version": "Fix-Version", "Value #B": "Erstmals gesehen" }
} }
],
"fieldConfig": { "defaults": {}, "overrides": [
{ "matcher": { "id": "byName", "options": "Erstmals gesehen" }, "properties": [ { "id": "unit", "value": "dateTimeAsIso" } ] },
{ "matcher": { "id": "byName", "options": "CVE" }, "properties": [ { "id": "links", "value": [ { "title": "NVD (Mitigation/Details)", "targetBlank": true, "url": "https://nvd.nist.gov/vuln/detail/${__value.text}" } ] } ] }
] }
},
{
"type": "state-timeline", "title": "Zeitstrahl: offene HIGH/CRITICAL je Target", "id": 6,
"gridPos": { "x": 0, "y": 16, "w": 24, "h": 8 },
"targets": [ { "expr": "count by (target) (trivy_vuln_info)", "legendFormat": "{{target}}", "refId": "A" } ],
"fieldConfig": { "defaults": { "custom": { "fillOpacity": 70 } }, "overrides": [] }
},
{
"type": "timeseries", "title": "Funde je Severity über Zeit", "id": 7,
"gridPos": { "x": 0, "y": 24, "w": 24, "h": 8 },
"targets": [ { "expr": "sum by (severity) (trivy_vuln_count)", "legendFormat": "{{severity}}", "refId": "A" } ]
}
]
}
+29
View File
@@ -51,3 +51,32 @@ groups:
severity: critical
annotations:
summary: "{{ $labels.instance }}: <10% Platz auf / frei"
# CVE-Pipeline (gitops#47): Funde aus den Trivy-Scans. Pflichtfelder je Alarm:
# CVE-ID, Mitigation (Fix-Version + NVD-Link), Ort (host), Typ (target_type);
# der Zeitstrahl kommt aus trivy_vuln_first_seen_timestamp + resolved-Edit.
# Alle Regeln routen per room-Label in den Security-Raum.
- name: axion-cve
rules:
- alert: TrivyCriticalVuln
expr: trivy_vuln_info{severity="CRITICAL"} == 1
labels:
severity: critical
room: security
annotations:
summary: "{{ $labels.cve }} (CRITICAL) in {{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}] — Paket {{ $labels.pkg }} {{ $labels.installed }}{{ if $labels.fixed_version }}, Fix: {{ $labels.fixed_version }}{{ else }}, noch kein Fix{{ end }} — https://nvd.nist.gov/vuln/detail/{{ $labels.cve }}"
- alert: TrivyHighVuln
expr: trivy_vuln_info{severity="HIGH"} == 1
for: 24h
labels:
severity: warning
room: security
annotations:
summary: "{{ $labels.cve }} (HIGH) in {{ $labels.target }} [{{ $labels.target_type }}@{{ $labels.host }}] — Paket {{ $labels.pkg }} {{ $labels.installed }}{{ if $labels.fixed_version }}, Fix: {{ $labels.fixed_version }}{{ else }}, noch kein Fix{{ end }} — https://nvd.nist.gov/vuln/detail/{{ $labels.cve }}"
- alert: TrivyScanStale
expr: time() - trivy_last_scan_timestamp > 172800
labels:
severity: warning
room: security
annotations:
summary: "CVE-Scan fuer {{ $labels.target }} ist aelter als 2 Tage — Scanner pruefen (ein still gestorbener Scanner macht blind)"
+5
View File
@@ -47,3 +47,8 @@ scrape_configs:
- job_name: "gameserver_cadvisor"
static_configs:
- targets: ["157.90.155.206:8080"]
# CVE-Exporter (gitops#47)
- job_name: "cve_exporter"
static_configs:
- targets: ["cve-exporter:9101"]