Merge branch 'rework/monitoring'

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

monitoring/.env.example

@@ -0,0 +1,4 @@
+# Vorlage: nach .env kopieren und Werte setzen.
+# Werte in Single-Quotes, damit Sonderzeichen shell-safe sind.
+GRAFANA_ADMIN_USER='admin'
+GRAFANA_ADMIN_PASSWORD='changeme'

monitoring/README.md

@@ -0,0 +1,54 @@
+# Monitoring-Stack (Operating-Host)
+
+Prometheus + Loki + Grafana + Alloy + node-exporter fuer den Operating-Server
+(`CFGMON`). Vollstaendig provisioniert: Datasources und Dashboards liegen als
+Code in diesem Repo, ein `docker compose up -d` auf einem frischen Host stellt
+den kompletten Stack wieder her.
+
+## Deployment
+
+```bash
+cp .env.example .env   # Werte setzen (nur beim allerersten Start relevant)
+docker network create traefik   # falls noch nicht vorhanden
+docker compose up -d
+```
+
+## Struktur
+
+| Pfad | Inhalt |
+|---|---|
+| `docker-compose.yml` | Stack-Definition, Images auf feste Versionen gepinnt |
+| `prometheus/prometheus.yml` | Scrape-Konfiguration |
+| `loki/loki-config.yaml` | Loki Single-Binary-Konfiguration, 90 Tage Retention |
+| `alloy/config.alloy` | Docker-Log-Collection -> Loki, node-exporter -> Prometheus |
+| `grafana/provisioning/` | Datasources (feste UIDs!) und Dashboard-Provider |
+| `grafana/dashboards/<ordner>/*.json` | Dashboards, je Unterordner ein Grafana-Ordner |
+
+## Externe Abhaengigkeiten
+
+- **Remote-Write eingehend** auf `:9090`: k3s-Cluster und Matrix-Server pushen
+  Metriken (`flux`, `kube_state_metrics`, `synapse`).
+- **Log-Push eingehend** auf `:3100` (Loki) von externen Alloys.
+- **Scrape ausgehend**: k3s-Host (`10.0.0.2:9100`), Pterodactyl-Host
+  (`157.90.155.206:9100` + `:8080`).
+- `cadvisor` und `traefik` werden gescraped, laufen aber in **anderen** Stacks
+  (cadvisor: Portainer-Stack `thread-net-git`; beide haengen im externen
+  `traefik`-Netz).
+
+## Dashboards aendern
+
+Provisionierte Dashboards sind in der UI editierbar (`allowUiUpdates: true`),
+aber die JSON-Datei im Repo ist die Quelle der Wahrheit: UI-Aenderungen
+muessen exportiert und committet werden, sonst gehen sie beim naechsten
+Datei-Update verloren.
+
+## Offene Punkte / Sicherheit
+
+- `9090`, `3100`, `9100` sind auf der oeffentlichen IP ohne Auth erreichbar
+  (Docker umgeht ufw). Per **Hetzner Cloud Firewall** auf die IPs der
+  bekannten Absender einschraenken.
+- Pterodactyl-Host `157.90.155.206`: node-exporter und cadvisor antworten
+  aktuell nicht (Targets down).
+- Loki `table_manager` ist deprecated; Retention-Enforcement laeuft aktuell
+  nicht ueber den Compactor -- bei Gelegenheit auf Compactor-Retention
+  umstellen.

monitoring/alloy/config.alloy

@@ -1,10 +1,33 @@
+// Log- und Metrik-Collector fuer den Operating-Host.
+
 discovery.docker "linux_containers" {
   host = "unix:///var/run/docker.sock"
 }
 
+// Container-Metadaten als Loki-Labels (additiv zu job/instance/service_name).
+discovery.relabel "docker_containers" {
+  targets = discovery.docker.linux_containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
+    target_label  = "compose_project"
+  }
+}
+
 loki.source.docker "docker_logs" {
-  host       = "unix:///var/run/docker.sock" // Diesen Wert habe ich hinzugefügt
+  host       = "unix:///var/run/docker.sock"
-  targets    = discovery.docker.linux_containers.targets
+  targets    = discovery.relabel.docker_containers.output
   forward_to = [loki.write.local_loki.receiver]
 }
 
@@ -15,7 +38,7 @@ loki.write "local_loki" {
 }
 
 prometheus.scrape "node_exporter" {
-  targets = [{"__address__" = "node-exporter:9100"}]
+  targets    = [{"__address__" = "node-exporter:9100"}]
   forward_to = [prometheus.remote_write.local_prometheus.receiver]
 }
 

monitoring/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   prometheus:
-    image: prom/prometheus:latest
+    image: prom/prometheus:v3.3.1
     container_name: prometheus
     restart: unless-stopped
     volumes:
@@ -11,6 +11,9 @@ services:
       - '--storage.tsdb.path=/prometheus'
       - '--storage.tsdb.retention.time=45d'
       - '--storage.tsdb.retention.size=15GB'
+      # Remote-Write-Receiver: k3s-Cluster und Matrix-Server pushen hierher.
+      # ACHTUNG: Port 9090 ist oeffentlich und ohne Auth erreichbar --
+      # per Hetzner Cloud Firewall auf die Absender-IPs einschraenken!
       - '--web.enable-remote-write-receiver'
     ports:
       - "9090:9090"
@@ -18,20 +21,22 @@ services:
       - traefik
 
   loki:
-    image: grafana/loki:latest
+    image: grafana/loki:3.7.1
     container_name: loki
     restart: unless-stopped
     volumes:
       - ./loki/loki-config.yaml:/etc/loki/local-config.yaml:ro
       - loki_data:/loki
     command: -config.file=/etc/loki/local-config.yaml
+    # Port 3100 oeffentlich: externe Alloys pushen Logs hierher.
+    # Ebenfalls per Firewall auf bekannte Absender-IPs einschraenken.
     ports:
       - "3100:3100"
     networks:
       - traefik
 
   alloy:
-    image: grafana/alloy:latest
+    image: grafana/alloy:v1.16.0
     container_name: alloy
     restart: unless-stopped
     volumes:
@@ -50,14 +55,18 @@ services:
       - loki
 
   grafana:
-    image: grafana/grafana:latest
+    image: grafana/grafana:12.0.0
     container_name: grafana
     restart: unless-stopped
     environment:
-      - GF_SECURITY_ADMIN_USER=${Grafana_ADMIN}
+      # Greift nur beim allerersten Start mit leerem Volume.
-      - GF_SECURITY_ADMIN_PASSWORD=${Grafana_ADMIN_PW}
+      # Der Live-Admin ist davon unabhaengig (Passwortaenderungen in der UI).
+      - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER}
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD}
     volumes:
       - grafana_data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
     labels:
       - "traefik.enable=true"
       - "traefik.docker.network=traefik"
@@ -72,7 +81,7 @@ services:
       - traefik
 
   node-exporter:
-    image: prom/node-exporter:latest
+    image: prom/node-exporter:v1.9.1
     container_name: node-exporter
     restart: unless-stopped
     volumes:

monitoring/grafana/dashboards/gameserver/pterodactyl-server.json

@@ -0,0 +1,970 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 16575,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 14,
+      "panels": [],
+      "title": "CPU & Uptime",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "belof8d814dtse"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "expr": "pterodactyl_server_cpu_absolute{server_name=~\"$server\"}",
+          "legendFormat": "Server: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "CPU usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "belof8d814dtse"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "green",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "ms",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 22,
+      "options": {
+        "displayMode": "gradient",
+        "maxVizHeight": 300,
+        "minVizHeight": 16,
+        "minVizWidth": 8,
+        "namePlacement": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "sizing": "auto",
+        "valueMode": "color"
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "expr": "pterodactyl_server_uptime_milliseconds {server_name=~\"$server\"}",
+          "interval": "",
+          "legendFormat": "Name: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Uptime",
+      "type": "bargauge"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 7
+      },
+      "id": 16,
+      "panels": [],
+      "title": "Memory",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "belof8d814dtse"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percentunit",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "expr": "pterodactyl_server_memory_mebibyte {server_name=~\"$server\"} / pterodactyl_server_max_memory_mebibyte {server_name=~\"$server\"}",
+          "legendFormat": "Server: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Memory usage percent",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "belof8d814dtse"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "mbytes",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "expr": "pterodactyl_server_memory_mebibyte {server_name=~\"$server\"}",
+          "legendFormat": "Server: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Memory usage mebibytes",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 14
+      },
+      "id": 18,
+      "panels": [],
+      "title": "Disk",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "belof8d814dtse"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percentunit",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 15
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "expr": "pterodactyl_server_disk_mebibyte {server_name=~\"$server\"} / pterodactyl_server_max_disk_mebibyte {server_name=~\"$server\"}",
+          "legendFormat": "Server: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Disk usage percent",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "belof8d814dtse"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "mbytes",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 15
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "expr": "pterodactyl_server_disk_mebibyte {server_name=~\"$server\"}",
+          "legendFormat": "Server: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Disk usage mebibytes",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 21
+      },
+      "id": 20,
+      "panels": [],
+      "title": "Network",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "belof8d814dtse"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decmbytes",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 22
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "expr": "pterodactyl_server_network_rx_mebibyte {server_name=~\"$server\"}",
+          "legendFormat": "RX: Server: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "expr": "pterodactyl_server_network_tx_mebibyte {server_name=~\"$server\"}",
+          "hide": false,
+          "legendFormat": "TX: Server: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Network usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "belof8d814dtse"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "MBs",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 22
+      },
+      "id": 24,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.0.7",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "expr": "increase(pterodactyl_server_network_rx_mebibyte {server_name=~\"$server\"}[$__interval])",
+          "legendFormat": "RX: Server: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "expr": "increase(pterodactyl_server_network_mebibyte {server_name=~\"$server\"}[$__interval])",
+          "hide": false,
+          "legendFormat": "TX: Server: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Network traffic",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 28
+      },
+      "id": 26,
+      "panels": [],
+      "title": "Backup",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "belof8d814dtse"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red"
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "dateTimeAsLocalNoDateIfToday",
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 0,
+        "y": 29
+      },
+      "id": 28,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "vertical",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.3.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "belof8d814dtse"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "pterodactyl_server_most_recent_backup_time{server_name=~\"$server\"}*1000",
+          "instant": false,
+          "legendFormat": "Server: {{server_name}}, ID: {{id}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Most Recent Backup Time",
+      "type": "stat"
+    }
+  ],
+  "refresh": "1m",
+  "schemaVersion": 39,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "belof8d814dtse"
+        },
+        "definition": "label_values(pterodactyl_server_io, server_name)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Server",
+        "multi": true,
+        "name": "server",
+        "options": [],
+        "query": {
+          "query": "label_values(pterodactyl_server_io, server_name)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Pterodactyl Server",
+  "uid": "UJnT34C7k",
+  "version": 1,
+  "weekStart": ""
+}

monitoring/grafana/dashboards/general/logs-app.json

@@ -0,0 +1,247 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "$$hashKey": "object:75",
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Log Viewer Dashboard for Loki",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [
+    {
+      "$$hashKey": "object:59",
+      "icon": "bolt",
+      "includeVars": true,
+      "keepTime": true,
+      "tags": [],
+      "targetBlank": true,
+      "title": "View In Explore",
+      "type": "link",
+      "url": "/explore?orgId=1&left=[\"now-1h\",\"now\",\"Loki\",{\"expr\":\"{job=\\\"$app\\\"}\"},{\"ui\":[true,true,true,\"none\"]}]"
+    },
+    {
+      "$$hashKey": "object:61",
+      "icon": "external link",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Learn LogQL",
+      "type": "link",
+      "url": "https://grafana.com/docs/loki/latest/logql/"
+    }
+  ],
+  "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "cfk1pam8svmkgd"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "hidden",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "bars",
+            "fillOpacity": 100,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "tooltip": {
+          "hideZeros": false,
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.0.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "cfk1pam8svmkgd"
+          },
+          "expr": "sum(count_over_time({job=\"$app\"} |= \"$search\" [$__interval]))",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "cfk1pam8svmkgd"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 25,
+        "w": 24,
+        "x": 0,
+        "y": 3
+      },
+      "id": 2,
+      "maxDataPoints": "",
+      "options": {
+        "dedupStrategy": "none",
+        "enableInfiniteScrolling": false,
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "12.0.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "cfk1pam8svmkgd"
+          },
+          "expr": "{job=\"$app\"} |= \"$search\" | logfmt",
+          "hide": false,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "",
+      "transparent": true,
+      "type": "logs"
+    }
+  ],
+  "preload": false,
+  "refresh": false,
+  "schemaVersion": 41,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "datasource": "cfk1pam8svmkgd",
+        "definition": "label_values(job)",
+        "includeAll": false,
+        "label": "App",
+        "name": "app",
+        "options": [],
+        "query": "label_values(job)",
+        "refresh": 1,
+        "regex": "",
+        "type": "query"
+      },
+      {
+        "current": {
+          "text": "",
+          "value": ""
+        },
+        "label": "String Match",
+        "name": "search",
+        "options": [
+          {
+            "selected": true,
+            "text": "",
+            "value": ""
+          }
+        ],
+        "query": "",
+        "type": "textbox"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ]
+  },
+  "timezone": "",
+  "title": "Logs / App",
+  "uid": "sadlil-loki-apps-dashboard",
+  "version": 2
+}

monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,39 @@
+# Grafana Dashboard-Provisioning.
+# Ein Provider pro Ordner; folderUid entspricht den bereits existierenden
+# Ordnern in der laufenden Instanz, damit nichts doppelt angelegt wird.
+# allowUiUpdates: true -> Dashboards sind in der UI editierbar, aber:
+# Aenderungen muessen zurueck ins Repo exportiert werden, sonst werden sie
+# beim naechsten Aendern der JSON-Datei ueberschrieben.
+apiVersion: 1
+
+providers:
+  - name: operating
+    folder: Operating
+    folderUid: delomy5mz83r4a
+    type: file
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards/operating
+
+  - name: matrix
+    folder: matrix
+    folderUid: felomzoe0lq80d
+    type: file
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards/matrix
+
+  - name: gameserver
+    folder: Gamesserver
+    folderUid: bfk1ljwhp77r4d
+    type: file
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards/gameserver
+
+  - name: general
+    folder: ''
+    type: file
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards/general

monitoring/grafana/provisioning/datasources/datasources.yml

@@ -0,0 +1,21 @@
+# Grafana Datasource-Provisioning.
+# Die UIDs sind fix und werden von allen Dashboards referenziert -- nicht aendern!
+apiVersion: 1
+
+datasources:
+  - name: prometheus
+    uid: belof8d814dtse
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+    jsonData:
+      httpMethod: POST
+
+  - name: loki
+    uid: cfk1pam8svmkgd
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false

monitoring/prometheus/prometheus.yml

@@ -1,9 +1,10 @@
 global:
   scrape_interval: 15s
 
-# local metrics
+# Zusaetzlich zu diesen Scrape-Targets kommen Metriken per Remote-Write rein
+# (k3s-Cluster: flux, kube_state_metrics; Matrix-Server: synapse).
 scrape_configs:
-  ## operating server
+  ## Operating-Server (dieser Host)
   - job_name: "operating_prometheus"
     static_configs:
       - targets: ["localhost:9090"]
@@ -13,6 +14,7 @@ scrape_configs:
       - targets: ["node-exporter:9100"]
 
   - job_name: "operating_cadvisor"
+    # cadvisor laeuft im Portainer-Stack "thread-net-git", selbes traefik-Netz
     static_configs:
       - targets: ["cadvisor:8080"]
 
@@ -21,6 +23,7 @@ scrape_configs:
     static_configs:
       - targets: ["traefik:8080"]
 
+  ## Externe Hosts
   - job_name: "k3s_host_node"
     static_configs:
       - targets: ["10.0.0.2:9100"]
@@ -36,7 +39,3 @@ scrape_configs:
   - job_name: "gameserver_cadvisor"
     static_configs:
       - targets: ["157.90.155.206:8080"]
-  ## matrix server (auskommentiert, aber jetzt syntaktisch korrekt eingerückt)
-  # - job_name: "matrix_prometheus"
-  #   static_configs:
-  #     - targets: ["matrix-server-ip:9090"]