commit 00a7bc8b4b6faabe6b02c98b0e7bae15c9dbfd21
parent c9ac0f00086853910cb37795a27a712ea6eb3283
Author: Christian Grothoff <christian@grothoff.org>
Date: Mon, 20 Jan 2025 20:29:21 +0100
borg backup playbooks
Diffstat:
12 files changed, 248 insertions(+), 5 deletions(-)
diff --git a/README b/README
@@ -31,6 +31,33 @@ For production, replace the "test-secrets.yml" file with the actual secrets
for your deployment.
+## Setting up backups
+
+First run:
+
+./extract-borg-key.sh
+
+The resulting SSH public key should be added to the borg-account
+of the host storing the backup. The playbook contains the target
+hostname!
+
+Once the SSH key is deployed and the backup has been initialized
+server-side (see admin-logs/pixel/03-borg.txt), start the daily
+backups via:
+
+./start-borg-backups.sh
+
+This will make a backup basically everything relevant to the
+deployment, **except** the exchange online signing keys. The
+backup will in particular include the system configuration
+and a full (xz-compressed) snapshot of the database. Thus,
+the backups should also suffice to diagnose problems.
+
+Backups are set to retain daily snapshots of the last 7 days,
+weekly snapshots for the last 4 weeks, and monthly snapshots
+for the last 6 months.
+
+
## Running the import/export Playbooks (TOPS-only)
```
diff --git a/TODO b/TODO
@@ -3,8 +3,7 @@
=> https://github.com/FoxyRoles/ansible-dkim seems about right!
- playbooks/libeufin-export.yml: see FIXME, can you please fix that? ;-)
-@TBD (#9352)2
-- postgres prometheus exporter setup is incomplete: postgres part (access!) is missing
+@TBD (#9352)
- setup loki log aggregator
- setup HTTPS reverse proxy loki
- check limit access using basic auth to prometheus exporters
diff --git a/extract-borg-key.sh b/extract-borg-key.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -eu
+
+ansible-playbook --inventory inventories/tops --user root playbooks/borg-ssh-export.yml
+cat borg.pub/*/root/.ssh/borg.pub
+rm -rf borg.pub/
+exit 0
diff --git a/playbooks/borg-ssh-export.yml b/playbooks/borg-ssh-export.yml
@@ -0,0 +1,8 @@
+---
+- name: Export SSH public key used for backups
+ hosts: all
+ roles:
+ - borg-ssh-export
+ vars:
+ # Hostname where we will store backups
+ BORG_HOST: pixel.taler-systems.com
diff --git a/playbooks/borg-start.yml b/playbooks/borg-start.yml
@@ -0,0 +1,10 @@
+---
+- name: Start backups with borg
+ hosts: all
+ roles:
+ - borg-start
+ vars:
+ # Hostname where we will store backups
+ BORG_HOST: pixel.taler-systems.com
+ # Target for the backup (repo must exist and we must have SSH access).
+ BORG_REPO: "ssh://borg@{{ BORG_HOST }}/~/spec-backup"
diff --git a/roles/borg-ssh-export/tasks/main.yml b/roles/borg-ssh-export/tasks/main.yml
@@ -0,0 +1,27 @@
+---
+- name: Install Borg package
+ apt:
+ name:
+ - borgbackup
+ - xz-utils
+ state: latest
+ when: ansible_os_family == 'Debian'
+
+# This step should not be needed, how else did we log in.
+# That said, can't hurt and seems cleaner to have it.
+- name: Ensure /root/.ssh/ directory exists
+ file:
+ path: "/root/.ssh/"
+ state: directory
+
+- name: Create SSH key pair for use for backups by root
+ ansible.builtin.shell:
+ cmd: "ssh-keygen -P '' -t ed25519 -f /root/.ssh/borg"
+ creates: /root/.ssh/borg.pub
+ become: yes
+ become_user: root
+
+- name: Fetch file to local system
+ fetch:
+ src: "/root/.ssh/borg.pub"
+ dest: "../borg.pub"
diff --git a/roles/borg-start/tasks/main.yml b/roles/borg-start/tasks/main.yml
@@ -0,0 +1,42 @@
+---
+- name: Ensure /root/bin/ directory exists
+ file:
+ path: "/root/bin/"
+ state: directory
+
+- name: Place shell script to do backups
+ ansible.builtin.template:
+ src: templates/root/bin/borg-backup.sh
+ dest: /root/bin/borg-backup.sh
+ owner: root
+ group: root
+ mode: 0700
+
+- name: Check SSH key for backups exists
+ stat:
+ path: "/root/.ssh/borg"
+ register: have_ssh_key
+
+- name: Place ssh configuration
+ ansible.builtin.template:
+ src: templates/root/.ssh/config
+ dest: /root/.ssh/config
+ owner: root
+ group: root
+ mode: 0600
+
+- name: Add host key for borg server
+ ansible.builtin.shell:
+ cmd: ssh-keyscan {{ BORG_HOST }} >> .ssh/known_hosts
+
+- name: Fail if we do not have an SSH key for the backup server
+ fail:
+ msg: "You need to first run extract-borg-key.sh"
+ when: not have_ssh_key.stat.exists
+
+- name: Create cron job to run daily backups
+ ansible.builtin.cron:
+ name: "perform backup"
+ minute: "43"
+ hour: "2"
+ job: "/root/bin/borg-backup.sh"
diff --git a/roles/borg-start/templates/root/.ssh/config b/roles/borg-start/templates/root/.ssh/config
@@ -0,0 +1,6 @@
+Host {{ BORG_HOST }}
+ HostName {{ BORG_HOST }}
+ Port 22
+ User borg
+ IdentityFile ~/.ssh/borg
+ IdentitiesOnly yes
diff --git a/roles/borg-start/templates/root/bin/borg-backup.sh b/roles/borg-start/templates/root/bin/borg-backup.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+set -eu
+
+export BORG_REPO='{{ BORG_REPO }}'
+export BORG_PASSPHRASE='{{ BORG_PASSPHRASE }}'
+
+# some helpers and error handling:
+info() { printf "\n%s %s\n\n" "$( date )" "$*" >&2; }
+trap 'echo $( date ) Backup interrupted >&2; exit 2' INT TERM
+
+info "Dumping database"
+
+sudo -u postgres pg_dumpall --clean | xz -T0 > postgres-backup.sql.xz
+
+info "Starting backup"
+
+# Backup the most important directories into an archive named after
+# the machine this script is currently running on:
+
+borg create \
+ --verbose \
+ --filter AME \
+ --list \
+ --stats \
+ --show-rc \
+ --compression lz4 \
+ --exclude-caches \
+ --exclude 'home/*/.cache/*' \
+ --exclude 'var/tmp/*' \
+ --exclude 'var/lib/taler-exchange/secmod-*/*' \
+ \
+ ::'{hostname}-{now}' \
+ /etc \
+ /root \
+ /var/lib/libeufin-bank \
+ /var/lib/libeufin-nexus \
+ /var/lib/taler-auditor \
+ /var/lib/taler-exchange
+
+backup_exit=$?
+
+info "Removing database dump"
+
+rm postgres-backup.sql.xz
+
+
+info "Pruning repository"
+
+# Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly
+# archives of THIS machine. The '{hostname}-*' matching is very important to
+# limit prune's operation to this machine's archives and not apply to
+# other machines' archives also:
+
+borg prune \
+ --list \
+ --glob-archives '{hostname}-*' \
+ --show-rc \
+ --keep-daily 7 \
+ --keep-weekly 4 \
+ --keep-monthly 6
+
+prune_exit=$?
+
+# actually free repo disk space by compacting segments
+
+info "Compacting repository"
+
+borg compact
+
+compact_exit=$?
+
+# use highest exit code as global exit code
+global_exit=$(( backup_exit > prune_exit ? backup_exit : prune_exit ))
+global_exit=$(( compact_exit > global_exit ? compact_exit : global_exit ))
+
+if [ ${global_exit} -eq 0 ]; then
+ info "Backup, Prune, and Compact finished successfully"
+elif [ ${global_exit} -eq 1 ]; then
+ info "Backup, Prune, and/or Compact finished with warnings"
+else
+ info "Backup, Prune, and/or Compact finished with errors"
+fi
+
+exit ${global_exit}
diff --git a/roles/monitoring/handlers/main.yml b/roles/monitoring/handlers/main.yml
@@ -8,3 +8,13 @@
service:
name: prometheus-postgres-exporter
state: restarted
+
+- name: restart node-exporter
+ service:
+ name: prometheus-node-exporter
+ state: restarted
+
+- name: restart nginx-exporter
+ service:
+ name: prometheus-nginx-exporter
+ state: restarted
diff --git a/roles/monitoring/tasks/main.yml b/roles/monitoring/tasks/main.yml
@@ -89,16 +89,25 @@
- name: Create prometheus database user
community.postgresql.postgresql_user:
name: prometheus
+ become: yes
+ become_user: postgres
- name: Grant access to postgres database to the postgres-exporter
become: yes
become_user: postgres
community.postgresql.postgresql_query:
login_user: postgres
+ db: postgres
query:
- - "SELECT format('GRANT CONNECT ON DATABASE %I TO prometheus;', datname) FROM pg_database \gexec"
- - "GRANT USAGE ON SCHEMA pg_catalog TO prometheus;"
- - "GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO prometheus;"
+ GRANT CONNECT ON DATABASE "challenger-email" TO prometheus;
+ GRANT CONNECT ON DATABASE "challenger-postal" TO prometheus;
+ GRANT CONNECT ON DATABASE "challenger-sms" TO prometheus;
+ GRANT CONNECT ON DATABASE libeufin TO prometheus;
+ GRANT CONNECT ON DATABASE postgres TO prometheus;
+ GRANT CONNECT ON DATABASE "taler-auditor" TO prometheus;
+ GRANT CONNECT ON DATABASE "taler-exchange" TO prometheus;
+ GRANT USAGE ON SCHEMA pg_catalog TO prometheus;
+ GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO prometheus;
- name: Configure node-exporter
copy:
diff --git a/start-borg-backups.sh b/start-borg-backups.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -eu
+
+if [ -z ${BORG_PASSPHRASE:-} ]
+then
+ echo "You need to set the BORG_PASSPHRASE in your environment before running this script!"
+ exit 1
+fi
+
+ansible-playbook --verbose --extra-vars BORG_PASSPHRASE="$BORG_PASSPHRASE" --inventory inventories/tops --user root playbooks/borg-start.yml
+
+exit 0
