feat(backup): monthly restore drill that proves the backups are restorable
Restores the Borg archives into a throwaway postgres inside the pod and passes only when rows actually land — the pg_restore exit code is not proof, counted rows are. Production is never touched; the repos are only read. Automated rather than a documented cadence: a check nobody performs is the same mistake as an untested backup, one level up. Runs on the 4th at 04:20, after the nightly jobs. Verified manually before commit (synapse 31908 rows, MAS 16085, wiki 251). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Thore Cimbal
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
5f54fbef8e
commit
b61dfd968e
@@ -40,6 +40,8 @@ resources:
|
||||
# Backup zur Hetzner Storage Box (Issues #6 + #15)
|
||||
- synapse-backup-secret.yaml
|
||||
- synapse-backup.yaml
|
||||
# Monatliche Restore-Probe: spielt die Sicherungen isoliert zurueck (#0030)
|
||||
- restore-drill.yaml
|
||||
# Automatisierte TURN-Secret-Rotation (Issue #38)
|
||||
- turn-secret-rotation-secret.yaml
|
||||
- turn-secret-rotation.yaml
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
# Monatliche Restore-Probe (#0030): spielt die Borg-Sicherungen in eine Wegwerf-Postgres
|
||||
# IM POD zurueck und besteht nur, wenn wirklich Zeilen ankommen. Die Produktion wird nicht
|
||||
# angefasst - es wird ausschliesslich aus den Repos gelesen.
|
||||
#
|
||||
# Warum automatisch statt "einmal im Quartal dran denken": eine Sicherung, die nie
|
||||
# zurueckgespielt wurde, ist eine Vermutung (#0030) - ein Pruefrhythmus, den niemand
|
||||
# ausfuehrt, ist genau derselbe Fehler eine Ebene hoeher.
|
||||
#
|
||||
# Abgedeckt sind die unersetzlichen Daten: synapse + matrixauthenticationservice und das
|
||||
# Wiki. Authentik ist bewusst nicht Teil des automatischen Laufs (Flows/Provider liegen
|
||||
# als Blueprints deklarativ im Repo, die DB ist also weitgehend reproduzierbar); dafuer
|
||||
# gibt es die Stufe 3 in notfallhandbuch/notfall.sh auf Zuruf.
|
||||
#
|
||||
# Fehlschlag = der Job schlaegt fehl. Alarmierung dazu: threadnet-operating,
|
||||
# monitoring/prometheus/alerts.yml (BackupJobFailed / RestoreDrillStale).
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: restore-drill
|
||||
namespace: matrix
|
||||
spec:
|
||||
# 4. des Monats, 04:20 - deutlich nach den naechtlichen Backups (03:00/03:15/03:30),
|
||||
# damit die Probe den frischen Stand zieht.
|
||||
schedule: "20 4 4 * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: synapse-backup # NetworkPolicy/Egress wie die Backups
|
||||
app.kubernetes.io/component: restore-drill
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: drill
|
||||
image: rohana.axion1337.de/sorb/axion-backup:v2
|
||||
env:
|
||||
- name: BORG_PASSPHRASE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: synapse-backup-credentials
|
||||
key: borg-passphrase
|
||||
- name: SSH_PRIVATE_KEY_FILE
|
||||
value: /secrets/ssh/ssh-private-key
|
||||
- name: SSH_KNOWN_HOSTS_FILE
|
||||
value: /secrets/known-hosts/known_hosts
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -eu
|
||||
export BORG_RSH="ssh -i $SSH_PRIVATE_KEY_FILE -o UserKnownHostsFile=$SSH_KNOWN_HOSTS_FILE -o StrictHostKeyChecking=yes"
|
||||
export PGDATA=/scratch/pgdata
|
||||
mkdir -p "$PGDATA" /scratch/sock
|
||||
chown postgres:postgres "$PGDATA" /scratch/sock
|
||||
echo "[drill] Wegwerf-Postgres starten"
|
||||
gosu postgres initdb -U postgres >/dev/null 2>&1
|
||||
gosu postgres pg_ctl -D "$PGDATA" -o "-k /scratch/sock -h ''" -w start >/dev/null
|
||||
# repo:datenbanken
|
||||
for pair in "synapse-backup:synapse matrixauthenticationservice" "wikijs-backup:wiki"; do
|
||||
repo="${pair%%:*}"; dbs="${pair#*:}"
|
||||
export BORG_REPO="ssh://u641795@u641795.your-storagebox.de:23/./${repo}"
|
||||
cd /scratch && rm -rf scratch
|
||||
latest=$(borg list --last 1 --format '{archive}' "$BORG_REPO")
|
||||
echo "[drill] ${repo}: Archiv ${latest}"
|
||||
borg extract "$BORG_REPO::${latest}" scratch/dumps
|
||||
for db in $dbs; do
|
||||
# pg_restore/psql laufen als root und verbinden per Socket als
|
||||
# postgres (initdb setzt lokal 'trust') - sonst scheitert das Lesen
|
||||
# der root-eigenen Dump-Dateien.
|
||||
dropdb -h /scratch/sock -U postgres --if-exists "$db"
|
||||
createdb -h /scratch/sock -U postgres "$db"
|
||||
rc=0
|
||||
pg_restore -h /scratch/sock -U postgres -d "$db" --no-owner \
|
||||
--no-privileges "scratch/dumps/${db}.dump" >/scratch/restore.log 2>&1 || rc=$?
|
||||
[ "$rc" != "0" ] && { echo "[drill] pg_restore Code $rc:"; tail -3 /scratch/restore.log; }
|
||||
psql -h /scratch/sock -U postgres -d "$db" -q -c 'ANALYZE;' >/dev/null 2>&1
|
||||
rows=$(psql -h /scratch/sock -U postgres -d "$db" -tA \
|
||||
-c 'select coalesce(sum(n_live_tup),0) from pg_stat_user_tables;')
|
||||
echo "[drill] ${db}: ${rows} Zeilen"
|
||||
# Der Exitcode von pg_restore ist kein Beweis (es warnt gern) -
|
||||
# gezaehlte Zeilen sind einer.
|
||||
if [ "$rows" -le 0 ]; then
|
||||
echo "[drill] FEHLER: ${db} ist nach dem Restore LEER - Sicherung unbrauchbar!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
echo "[drill] OK - alle Sicherungen liessen sich zurueckspielen"
|
||||
volumeMounts:
|
||||
- name: ssh-key
|
||||
mountPath: /secrets/ssh
|
||||
readOnly: true
|
||||
- name: known-hosts
|
||||
mountPath: /secrets/known-hosts
|
||||
readOnly: true
|
||||
- name: scratch
|
||||
mountPath: /scratch
|
||||
resources:
|
||||
requests:
|
||||
memory: 256Mi
|
||||
cpu: 100m
|
||||
limits:
|
||||
memory: 1Gi
|
||||
volumes:
|
||||
- name: ssh-key
|
||||
secret:
|
||||
secretName: synapse-backup-credentials
|
||||
defaultMode: 0400
|
||||
items:
|
||||
- key: ssh-private-key
|
||||
path: ssh-private-key
|
||||
- name: known-hosts
|
||||
configMap:
|
||||
name: synapse-backup-known-hosts
|
||||
- name: scratch
|
||||
emptyDir:
|
||||
sizeLimit: 4Gi
|
||||
Reference in New Issue
Block a user