sorb/threadnet-operating
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cve: four rules and three panels guard the derivation (#0106)

Browse Source 
Gate 3 planned three rules; there are four. The fourth covers a case the others
miss entirely: every source answers cleanly but empty. Then nothing is missing,
because the desired set is empty, the freshness stamps are current, and nothing
is scanned at all. The Python suite already carries that case as "an empty set
is not the same as success", so the rule belongs with it.

These are the first rule unit tests in this stack. Each rule has a case where it
must fire and one where it must stay silent, because a rule that always fires
cannot be told from a correct one otherwise. Two sabotages confirm the tests
bite: an unreachable threshold on the source-freshness rule makes the expected
alert vanish, and removing the six hour grace period makes the missing-targets
rule fire at five hours where the test demands silence.

The grace period is not padding. A full round over roughly 65 images takes time,
so right after a deploy the gap is real rather than wrong.

The dashboard gains coverage and unscanned-image counters in the two free slots
of the top row, and a source-freshness bar at the bottom, so no existing panel
moves. That bar is the only place where a failed derivation can be told apart
from success.
This commit is contained in:
Thore Cimbal
2026-08-21 12:00:00 +00:00
parent 27770b6ec7
commit b989987d69
3 changed files with 599 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
monitoring/grafana/dashboards/security/cve-overview.json
+402 -45
View File
@@ -1,64 +1,421 @@
{
"title": "CVE-Übersicht (Trivy)",
"uid": "cve-overview",
"tags": ["security", "cve", "gitops-47"],
"tags": [
"security",
"cve",
"gitops-47"
],
"timezone": "browser",
"schemaVersion": 39,
"refresh": "5m",
"time": { "from": "now-30d", "to": "now" },
"time": {
"from": "now-30d",
"to": "now"
},
"panels": [
{
"type": "stat", "title": "Offene CRITICAL", "id": 1,
"gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 },
"fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] },
"targets": [ { "expr": "sum(trivy_vuln_count{severity=\"CRITICAL\"}) or vector(0)", "instant": true, "refId": "A" } ]
},
{
"type": "stat", "title": "Offene HIGH", "id": 2,
"gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 },
"fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] } }, "overrides": [] },
"targets": [ { "expr": "sum(trivy_vuln_count{severity=\"HIGH\"}) or vector(0)", "instant": true, "refId": "A" } ]
},
{
"type": "stat", "title": "MEDIUM/LOW (Summe)", "id": 3,
"gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 },
"targets": [ { "expr": "sum(trivy_vuln_count{severity=~\"MEDIUM|LOW\"}) or vector(0)", "instant": true, "refId": "A" } ]
},
{
"type": "stat", "title": "Ältester Scan (Stunden)", "id": 4,
"gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 },
"fieldConfig": { "defaults": { "unit": "h", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 48 } ] } }, "overrides": [] },
"targets": [ { "expr": "(time() - min(trivy_last_scan_timestamp)) / 3600", "instant": true, "refId": "A" } ]
},
{
"type": "table", "title": "Offene CVEs (HIGH/CRITICAL) — CVE · Ort · Typ · Paket · Fix · seit", "id": 5,
"gridPos": { "x": 0, "y": 4, "w": 24, "h": 12 },
"type": "stat",
"title": "Offene CRITICAL",
"id": 1,
"gridPos": {
"x": 0,
"y": 0,
"w": 4,
"h": 4
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"targets": [
{ "expr": "trivy_vuln_info", "instant": true, "format": "table", "refId": "A" },
{ "expr": "trivy_vuln_first_seen_timestamp * 1000", "instant": true, "format": "table", "refId": "B" }
{
"expr": "sum(trivy_vuln_count{severity=\"CRITICAL\"}) or vector(0)",
"instant": true,
"refId": "A"
}
]
},
{
"type": "stat",
"title": "Offene HIGH",
"id": 2,
"gridPos": {
"x": 4,
"y": 0,
"w": 4,
"h": 4
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 1
}
]
}
},
"overrides": []
},
"targets": [
{
"expr": "sum(trivy_vuln_count{severity=\"HIGH\"}) or vector(0)",
"instant": true,
"refId": "A"
}
]
},
{
"type": "stat",
"title": "MEDIUM/LOW (Summe)",
"id": 3,
"gridPos": {
"x": 8,
"y": 0,
"w": 4,
"h": 4
},
"targets": [
{
"expr": "sum(trivy_vuln_count{severity=~\"MEDIUM|LOW\"}) or vector(0)",
"instant": true,
"refId": "A"
}
]
},
{
"type": "stat",
"title": "Ältester Scan (Stunden)",
"id": 4,
"gridPos": {
"x": 12,
"y": 0,
"w": 4,
"h": 4
},
"fieldConfig": {
"defaults": {
"unit": "h",
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 48
}
]
}
},
"overrides": []
},
"targets": [
{
"expr": "(time() - min(trivy_last_scan_timestamp)) / 3600",
"instant": true,
"refId": "A"
}
]
},
{
"type": "table",
"title": "Offene CVEs (HIGH/CRITICAL) — CVE · Ort · Typ · Paket · Fix · seit",
"id": 5,
"gridPos": {
"x": 0,
"y": 4,
"w": 24,
"h": 12
},
"targets": [
{
"expr": "trivy_vuln_info",
"instant": true,
"format": "table",
"refId": "A"
},
{
"expr": "trivy_vuln_first_seen_timestamp * 1000",
"instant": true,
"format": "table",
"refId": "B"
}
],
"transformations": [
{ "id": "merge", "options": {} },
{ "id": "organize", "options": {
"excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true, "Value #A": true },
"renameByName": { "cve": "CVE", "severity": "Severity", "host": "Ort (Host)", "target_type": "Typ", "target": "Target", "pkg": "Paket", "installed": "Installiert", "fixed_version": "Fix-Version", "Value #B": "Erstmals gesehen" }
} }
{
"id": "merge",
"options": {}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"__name__": true,
"job": true,
"instance": true,
"Value #A": true
},
"renameByName": {
"cve": "CVE",
"severity": "Severity",
"host": "Ort (Host)",
"target_type": "Typ",
"target": "Target",
"pkg": "Paket",
"installed": "Installiert",
"fixed_version": "Fix-Version",
"Value #B": "Erstmals gesehen"
}
}
}
],
"fieldConfig": { "defaults": {}, "overrides": [
{ "matcher": { "id": "byName", "options": "Erstmals gesehen" }, "properties": [ { "id": "unit", "value": "dateTimeAsIso" } ] },
{ "matcher": { "id": "byName", "options": "CVE" }, "properties": [ { "id": "links", "value": [ { "title": "NVD (Mitigation/Details)", "targetBlank": true, "url": "https://nvd.nist.gov/vuln/detail/${__value.text}" } ] } ] }
] }
"fieldConfig": {
"defaults": {},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Erstmals gesehen"
},
"properties": [
{
"id": "unit",
"value": "dateTimeAsIso"
}
]
},
{
"matcher": {
"id": "byName",
"options": "CVE"
},
"properties": [
{
"id": "links",
"value": [
{
"title": "NVD (Mitigation/Details)",
"targetBlank": true,
"url": "https://nvd.nist.gov/vuln/detail/${__value.text}"
}
]
}
]
}
]
}
},
{
"type": "state-timeline", "title": "Zeitstrahl: offene HIGH/CRITICAL je Target", "id": 6,
"gridPos": { "x": 0, "y": 16, "w": 24, "h": 8 },
"targets": [ { "expr": "count by (target) (trivy_vuln_info)", "legendFormat": "{{target}}", "refId": "A" } ],
"fieldConfig": { "defaults": { "custom": { "fillOpacity": 70 } }, "overrides": [] }
"type": "state-timeline",
"title": "Zeitstrahl: offene HIGH/CRITICAL je Target",
"id": 6,
"gridPos": {
"x": 0,
"y": 16,
"w": 24,
"h": 8
},
"targets": [
{
"expr": "count by (target) (trivy_vuln_info)",
"legendFormat": "{{target}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"fillOpacity": 70
}
},
"overrides": []
}
},
{
"type": "timeseries", "title": "Funde je Severity über Zeit", "id": 7,
"gridPos": { "x": 0, "y": 24, "w": 24, "h": 8 },
"targets": [ { "expr": "sum by (severity) (trivy_vuln_count)", "legendFormat": "{{severity}}", "refId": "A" } ]
"type": "timeseries",
"title": "Funde je Severity über Zeit",
"id": 7,
"gridPos": {
"x": 0,
"y": 24,
"w": 24,
"h": 8
},
"targets": [
{
"expr": "sum by (severity) (trivy_vuln_count)",
"legendFormat": "{{severity}}",
"refId": "A"
}
]
},
{
"type": "stat",
"title": "Deckung der Zielmenge",
"id": 8,
"gridPos": {
"x": 16,
"y": 0,
"w": 4,
"h": 4
},
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "orange",
"value": 0.9
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"targets": [
{
"expr": "cve_target_coverage_ratio",
"instant": true,
"refId": "A"
}
]
},
{
"type": "stat",
"title": "Ungeprüfte Images",
"id": 9,
"gridPos": {
"x": 20,
"y": 0,
"w": 4,
"h": 4
},
"fieldConfig": {
"defaults": {
"unit": "none",
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"targets": [
{
"expr": "cve_targets_missing or vector(0)",
"instant": true,
"refId": "A"
}
]
},
{
"type": "bargauge",
"title": "Frische der Zielquellen (Sekunden seit letzter Herleitung)",
"id": 10,
"gridPos": {
"x": 0,
"y": 32,
"w": 24,
"h": 6
},
"description": "Faellt eine Quelle aus, wird die Soll-Menge kleiner - und die Deckung sieht dadurch BESSER aus statt schlechter. Diese Zeile ist die einzige Stelle, an der ein Ausfall der Herleitung von Erfolg zu unterscheiden ist (ADR-0026).",
"fieldConfig": {
"defaults": {
"unit": "s",
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 7200
},
{
"color": "red",
"value": 10800
}
]
}
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal"
},
"targets": [
{
"expr": "cve_target_source_stale",
"instant": true,
"refId": "A",
"legendFormat": "{{quelle}}"
}
]
}
]
}
monitoring/prometheus/alerts.yml
+47
View File
@@ -106,6 +106,53 @@ groups:
annotations:
summary: "Der CVE-Exporter findet ueberhaupt keine Berichte mehr — der Scanner liefert nichts, die Schwachstellen-Sicht ist blind"
# --- Deckung der Zielmenge (#0106, ADR-0026) ---------------------
# Bis 2026-08-21 konnte die Pipeline nur sagen, WAS sie gescannt hat.
# Was sie nie angesehen hatte, war unsichtbar: 27 von 51 laufenden
# Images (52 %) - darunter der Web-Client, beide Traefik-Schichten und
# die Registry selbst. Die Zielliste war handgepflegt.
# 6h, weil eine volle Runde ueber ~65 Images dauert: nach einem
# Ausrollen ist die Luecke voruebergehend echt und kein Fehler.
- alert: CveTargetsMissing
expr: cve_targets_missing > 0
for: 6h
labels:
severity: warning
room: security
annotations:
summary: "{{ $value }} laufende Images werden NICHT auf CVEs geprueft — die Schwachstellen-Sicht hat ein Loch, das sie selbst nicht zeigt. Details: https://selendis.axion1337.de/d/cve-overview"
- alert: CveTargetsOrphaned
expr: cve_targets_orphaned > 0
for: 6h
labels:
severity: info
room: security
annotations:
summary: "{{ $value }} CVE-Bericht(e) betreffen Images, die nirgends laufen — Befunde, auf die niemand handeln kann (Rauschen, das wie Signal aussieht)"
# ⚠️ DIE WICHTIGSTE DER VIER. Faellt eine Quelle aus, wird die
# Soll-Menge KLEINER - und die Deckung saehe damit BESSER aus, nicht
# schlechter. Ein Ausfall waere ohne diese Regel von Erfolg nicht zu
# unterscheiden. Schwelle: 3x das Herleitungs-Intervall (1h).
- alert: CveTargetSourceStale
expr: cve_target_source_stale > 10800
labels:
severity: warning
room: security
annotations:
summary: "Die Quelle '{{ $labels.quelle }}' der CVE-Zielmenge ist seit {{ $value }}s nicht erneuert worden — die Zielmenge schrumpft still, und die Deckung sieht dadurch BESSER aus statt schlechter"
# Der Fall, den die drei darueber NICHT sehen: alle Quellen antworten
# sauber, aber leer. Dann ist 'missing' 0 (leere Soll-Menge) und
# 'stale' frisch - und es wird nichts mehr gescannt. In
# test_targets.py als "leere Menge ist NICHT dasselbe wie Erfolg".
- alert: CveTargetsNoneDesired
expr: cve_targets_desired_total == 0
for: 30m
labels:
severity: critical
room: security
annotations:
summary: "Die CVE-Zielmenge ist LEER — es wird nichts mehr geprueft, obwohl keine Quelle einen Fehler meldet"
# Sicherungen (management #0030). Bis 2026-08-14 gab es hierzu KEINE Regel: ein
# fehlgeschlagenes naechtliches Backup waere unbemerkt geblieben - genau der stille
# Ausfall, den das Issue beschreibt. Metriken kommen aus kube-state-metrics im
monitoring/prometheus/alerts_test.yml
+150
View File
@@ -0,0 +1,150 @@
# Regeltests fuer die Deckungs-Alarme (#0106).
# docker run --rm --entrypoint promtool -v "$PWD/prometheus:/p:ro" \
# prom/prometheus:v3.3.1 test rules /p/alerts_test.yml
#
# ⚠️ ZWECK: Eine Regel, die nur je gruen war, ist eine Vermutung. In diesem
# Projekt ist "meldet Erfolg, ist aber blind" die haeufigste Fehlerklasse
# (AAR 2026-08-01, Befund 1 und 3). Deshalb steht zu jeder Regel ein Fall, in
# dem sie feuern MUSS, und einer, in dem sie schweigen MUSS - sonst laesst sich
# eine Regel, die immer feuert, von einer richtigen nicht unterscheiden.
#
# Erste Regeltests in diesem Stack ueberhaupt; die uebrigen 16 Regeln sind
# bewusst nicht mit abgedeckt (Nicht-Ziel von #0106).
rule_files:
- alerts.yml
evaluation_interval: 1m
tests:
# --- CveTargetsMissing ----------------------------------------------------
- interval: 1m
name: "Luecke in der Deckung feuert nach 6h"
input_series:
- series: 'cve_targets_missing'
values: '24+0x480' # 8h lang 24 fehlende Images
alert_rule_test:
- eval_time: 5h
alertname: CveTargetsMissing
exp_alerts: [] # noch in der Karenz - eine volle Runde darf dauern
- eval_time: 7h
alertname: CveTargetsMissing
exp_alerts:
- exp_labels:
severity: warning
room: security
exp_annotations:
summary: "24 laufende Images werden NICHT auf CVEs geprueft — die Schwachstellen-Sicht hat ein Loch, das sie selbst nicht zeigt. Details: https://selendis.axion1337.de/d/cve-overview"
- interval: 1m
name: "Volle Deckung schweigt"
input_series:
- series: 'cve_targets_missing'
values: '0+0x480'
alert_rule_test:
- eval_time: 7h
alertname: CveTargetsMissing
exp_alerts: []
# --- CveTargetsOrphaned ---------------------------------------------------
- interval: 1m
name: "Berichte ohne laufendes Image feuern"
input_series:
- series: 'cve_targets_orphaned'
values: '2+0x480'
alert_rule_test:
- eval_time: 7h
alertname: CveTargetsOrphaned
exp_alerts:
- exp_labels:
severity: info
room: security
exp_annotations:
summary: "2 CVE-Bericht(e) betreffen Images, die nirgends laufen — Befunde, auf die niemand handeln kann (Rauschen, das wie Signal aussieht)"
# --- CveTargetSourceStale (die wichtigste) --------------------------------
- interval: 1m
name: "Eine eingefrorene Quelle faellt auf"
input_series:
# Die Registry-Quelle altert, die anderen beiden bleiben frisch.
- series: 'cve_target_source_stale{quelle="registry"}'
values: '0+60x400' # waechst um 60s je Minute -> nach 180 min > 10800
- series: 'cve_target_source_stale{quelle="cluster"}'
values: '30+0x400'
- series: 'cve_target_source_stale{quelle="operating"}'
values: '30+0x400'
alert_rule_test:
- eval_time: 2h
alertname: CveTargetSourceStale
exp_alerts: [] # 7200s < 10800s - noch in Ordnung
- eval_time: 5h
alertname: CveTargetSourceStale
exp_alerts:
- exp_labels:
severity: warning
room: security
quelle: registry # ⚠️ NUR die eingefrorene, nicht alle drei
exp_annotations:
summary: "Die Quelle 'registry' der CVE-Zielmenge ist seit 18000s nicht erneuert worden — die Zielmenge schrumpft still, und die Deckung sieht dadurch BESSER aus statt schlechter"
# --- CveTargetsNoneDesired ------------------------------------------------
# Der Fall, den die drei anderen NICHT sehen: alle Quellen antworten sauber,
# aber leer. missing = 0, stale frisch - und es wird nichts mehr geprueft.
- interval: 1m
name: "Leere Zielmenge trotz gesunder Quellen"
input_series:
- series: 'cve_targets_desired_total'
values: '0+0x120'
- series: 'cve_targets_missing'
values: '0+0x120'
- series: 'cve_target_source_stale{quelle="cluster"}'
values: '5+0x120'
alert_rule_test:
- eval_time: 20m
alertname: CveTargetsNoneDesired
exp_alerts: [] # Karenz 30m
- eval_time: 40m
alertname: CveTargetsNoneDesired
exp_alerts:
- exp_labels:
severity: critical
room: security
exp_annotations:
summary: "Die CVE-Zielmenge ist LEER — es wird nichts mehr geprueft, obwohl keine Quelle einen Fehler meldet"
# Gegenprobe im selben Lauf: die anderen drei schweigen hier, obwohl die
# Lage schlecht ist. Genau deshalb braucht es die vierte Regel.
- eval_time: 40m
alertname: CveTargetsMissing
exp_alerts: []
- eval_time: 40m
alertname: CveTargetSourceStale
exp_alerts: []
# --- Gegenprobe: gesunder Zustand loest KEINEN der vier Alarme aus --------
- interval: 1m
name: "Alles in Ordnung bleibt still"
input_series:
- series: 'cve_targets_missing'
values: '0+0x480'
- series: 'cve_targets_orphaned'
values: '0+0x480'
- series: 'cve_targets_desired_total'
values: '65+0x480'
- series: 'cve_target_source_stale{quelle="cluster"}'
values: '120+0x480'
- series: 'cve_target_source_stale{quelle="operating"}'
values: '120+0x480'
- series: 'cve_target_source_stale{quelle="registry"}'
values: '900+0x480'
alert_rule_test:
- eval_time: 7h
alertname: CveTargetsMissing
exp_alerts: []
- eval_time: 7h
alertname: CveTargetsOrphaned
exp_alerts: []
- eval_time: 7h
alertname: CveTargetsNoneDesired
exp_alerts: []
- eval_time: 7h
alertname: CveTargetSourceStale
exp_alerts: []