From c7c161ae991122525a27daf2176572902bc1f374 Mon Sep 17 00:00:00 2001 From: Boss Marco Date: Sun, 24 Apr 2022 16:00:06 +0200 Subject: rename to be clearer --- .../lib/systemd/system/taler-data-backup.service | 10 ------ .../usr/lib/systemd/system/taler-data-backup.timer | 10 ------ .../systemd/system/taler-prometheus-backup.service | 10 ++++++ .../systemd/system/taler-prometheus-backup.timer | 10 ++++++ experiment/scripts/data-backup.sh | 42 ---------------------- experiment/scripts/database.sh | 28 +++++++++------ experiment/scripts/monitor.sh | 2 +- experiment/scripts/prometheus-backup.sh | 42 ++++++++++++++++++++++ 8 files changed, 81 insertions(+), 73 deletions(-) delete mode 100644 configs/usr/lib/systemd/system/taler-data-backup.service delete mode 100644 configs/usr/lib/systemd/system/taler-data-backup.timer create mode 100644 configs/usr/lib/systemd/system/taler-prometheus-backup.service create mode 100644 configs/usr/lib/systemd/system/taler-prometheus-backup.timer delete mode 100755 experiment/scripts/data-backup.sh create mode 100755 experiment/scripts/prometheus-backup.sh diff --git a/configs/usr/lib/systemd/system/taler-data-backup.service b/configs/usr/lib/systemd/system/taler-data-backup.service deleted file mode 100644 index 304c9ab..0000000 --- a/configs/usr/lib/systemd/system/taler-data-backup.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Create periodic snapshots of the Prometheus and Loki data in the experiments -Wants=taler-data-backup.service - -[Service] -EnvironmentFile=/etc/environment -ExecStart=/root/scripts/data-backup.sh - -[Install] -WantedBy=multi-user.target diff --git a/configs/usr/lib/systemd/system/taler-data-backup.timer b/configs/usr/lib/systemd/system/taler-data-backup.timer deleted file mode 100644 index b49a4a1..0000000 --- a/configs/usr/lib/systemd/system/taler-data-backup.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Create periodic snapshots of the Prometheus and Loki data in the experiments -Requires=taler-data-backup.service - -[Timer] -Unit=taler-data-backup.service -OnCalendar=*:0/2 - -[Install] -WantedBy=timers.target diff --git a/configs/usr/lib/systemd/system/taler-prometheus-backup.service b/configs/usr/lib/systemd/system/taler-prometheus-backup.service new file mode 100644 index 0000000..8418b59 --- /dev/null +++ b/configs/usr/lib/systemd/system/taler-prometheus-backup.service @@ -0,0 +1,10 @@ +[Unit] +Description=Create periodic snapshots of the Prometheus data in the experiments +Wants=taler-prometheus-backup.timer + +[Service] +EnvironmentFile=/etc/environment +ExecStart=/root/scripts/prometheus-backup.sh + +[Install] +WantedBy=multi-user.target diff --git a/configs/usr/lib/systemd/system/taler-prometheus-backup.timer b/configs/usr/lib/systemd/system/taler-prometheus-backup.timer new file mode 100644 index 0000000..39607b2 --- /dev/null +++ b/configs/usr/lib/systemd/system/taler-prometheus-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Create periodic snapshots of the Prometheus data in the experiments +Requires=taler-prometheus-backup.service + +[Timer] +Unit=taler-prometheus-backup.service +OnCalendar=*:0/2 + +[Install] +WantedBy=timers.target diff --git a/experiment/scripts/data-backup.sh b/experiment/scripts/data-backup.sh deleted file mode 100755 index 08cd9aa..0000000 --- a/experiment/scripts/data-backup.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Script which creates a snapshot of a running Prometheus instance - -# This will copy the snapshot to the configured LOG_DIR -# (setup.sh) hopefully persistent on the Grid5000 NFS - -# Usage: ./data-backup.sh -# Used by taler-databackup.service - -set -eu - -if [[ $(ps -aux | grep "[data]-backup.sh" | wc -l) -eq 1 ]]; then - echo "Previous backup still running, skipping" - exit 0 -fi - -PROMETHEUS_DATA_DIR=/var/lib/prometheus/metrics2/snapshots - -SNAPSHOT_COPY_PATH=${EXPERIMENT_DATA_DIR}/prometheus-tmp -SNAPSHOT_PATH=${EXPERIMENT_DATA_DIR}/prometheus - -END_TIME=$(date +%s) - -SNAPSHOT="${PROMETHEUS_DATA_DIR}/$(\ - curl \ - -X POST \ - http://localhost:8080/api/v1/admin/tsdb/snapshot \ - | jq -r '.data.name' -)" - -# copy first because its nfs, when copied remove the previous one -cp -r "${SNAPSHOT}" "${SNAPSHOT_COPY_PATH}" -rm -rf "${SNAPSHOT_PATH}" -mv "${SNAPSHOT_COPY_PATH}" "${SNAPSHOT_PATH}" - -# export timestamps for the recover script in g5k-repo/additional -echo " -EXPERIMENT_START=${START_TIME} -SNAPSHOT_TIME=${END_TIME} -" > "${EXPERIMENT_DATA_DIR}/times.env" - diff --git a/experiment/scripts/database.sh b/experiment/scripts/database.sh index 47761fd..736efb9 100755 --- a/experiment/scripts/database.sh +++ b/experiment/scripts/database.sh @@ -17,17 +17,25 @@ cd /tmp # Don't use in combination with `setup_ram_storage` function setup_disks() { if [ -b /dev/disk1 ]; then - echo 'start=2048, type=83' | sfdisk /dev/disk1 || true - yes | mkfs.ext4 -m 0 /dev/disk1 || true - mkdir /mnt/disk || true - mount /dev/disk1 /mnt/disk || true - if [ ! -L /tmp/postgresql/${POSTGRES_VERSION}/main/pg_wal ]; then - rm -rf /mnt/disk/pg_wal || true - mv /tmp/postgresql/${POSTGRES_VERSION}/main/pg_wal/ /mnt/disk - ln -s /mnt/disk/pg_wal /tmp/postgresql/${POSTGRES_VERSION}/main/pg_wal - chown -R postgres:postgres /tmp/postgresql/${POSTGRES_VERSION}/main/pg_wal - fi + # Dahu ssd + DISK=/dev/disk1 + elif [ -b /dev/disk4 ]; then + # Yeti nvme + DISK=/dev/disk4 + else + return fi + echo 'start=2048, type=83' | sfdisk ${DISK} || true + yes | mkfs.ext4 -m 0 ${DISK} || true + mkdir /mnt/disk || true + mount ${DISK} /mnt/disk || true + if [ ! -L /tmp/postgresql/${POSTGRES_VERSION}/main/pg_wal ]; then + rm -rf /mnt/disk/pg_wal || true + mv /tmp/postgresql/${POSTGRES_VERSION}/main/pg_wal/ /mnt/disk + ln -s /mnt/disk/pg_wal /tmp/postgresql/${POSTGRES_VERSION}/main/pg_wal + chown -R postgres:postgres /tmp/postgresql/${POSTGRES_VERSION}/main/pg_wal + fi + } # Setup Postgres to use RAM instead of disks to store data diff --git a/experiment/scripts/monitor.sh b/experiment/scripts/monitor.sh index 7c62e75..94a21db 100755 --- a/experiment/scripts/monitor.sh +++ b/experiment/scripts/monitor.sh @@ -247,7 +247,7 @@ function init_monitor() { systemctl restart prometheus-postgres-exporter \ prometheus \ - taler-data-backup.timer + taler-prometheus-backup.timer exit 0 } diff --git a/experiment/scripts/prometheus-backup.sh b/experiment/scripts/prometheus-backup.sh new file mode 100755 index 0000000..08cd9aa --- /dev/null +++ b/experiment/scripts/prometheus-backup.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Script which creates a snapshot of a running Prometheus instance + +# This will copy the snapshot to the configured LOG_DIR +# (setup.sh) hopefully persistent on the Grid5000 NFS + +# Usage: ./data-backup.sh +# Used by taler-databackup.service + +set -eu + +if [[ $(ps -aux | grep "[data]-backup.sh" | wc -l) -eq 1 ]]; then + echo "Previous backup still running, skipping" + exit 0 +fi + +PROMETHEUS_DATA_DIR=/var/lib/prometheus/metrics2/snapshots + +SNAPSHOT_COPY_PATH=${EXPERIMENT_DATA_DIR}/prometheus-tmp +SNAPSHOT_PATH=${EXPERIMENT_DATA_DIR}/prometheus + +END_TIME=$(date +%s) + +SNAPSHOT="${PROMETHEUS_DATA_DIR}/$(\ + curl \ + -X POST \ + http://localhost:8080/api/v1/admin/tsdb/snapshot \ + | jq -r '.data.name' +)" + +# copy first because its nfs, when copied remove the previous one +cp -r "${SNAPSHOT}" "${SNAPSHOT_COPY_PATH}" +rm -rf "${SNAPSHOT_PATH}" +mv "${SNAPSHOT_COPY_PATH}" "${SNAPSHOT_PATH}" + +# export timestamps for the recover script in g5k-repo/additional +echo " +EXPERIMENT_START=${START_TIME} +SNAPSHOT_TIME=${END_TIME} +" > "${EXPERIMENT_DATA_DIR}/times.env" + -- cgit v1.2.3