blob: 12bddc5618c246f7ac077d7414f62705eec98281 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
#!/bin/bash
# Script to automatically fetch and clean data on the Grid'5000 shares
# Params:
# 1: Name for the experiment
# 2: Additional plot tar to include (see ./plots)
set -e
function work_in_progress() {
echo -ne '\r\033[K'
sleep 1
echo -ne '.\r'
sleep 1
echo -ne '..\r'
sleep 1
echo -ne '...\r'
sleep 1
}
function backup() {
if [ -z ${1} ]; then
echo "Missing experiment name"
exit 1
fi
if [ -z ${2} ]; then
echo "No plots will be included"
else
echo "The file '${2}' will be deleted when this script finishes, do you want to contiue? [y/n]"
while true; do
read -n 1 -s yn
case $yn in
[Yy]*) break 2;;
[Nn]*) exit ;;
*) echo "Please enter 'y' or 'n'" ;;
esac
done
fi
# Slugify the name
NAME=$(\
echo "$1" | \
sed -r s/[~\^]+//g | \
sed -r s/[^a-zA-Z0-9]+/-/g | \
sed -r s/^-+\|-+$//g | \
tr A-Z a-z \
)
BACKUP_NAME="${NAME}-$(date +%d-%m-%y).tar"
BACKUP_DIR="archives"
G5K_BACKUP="g5k-$(date +%s).tar.gz"
ssh g5k 'rm -rf grenoble/g5k.tar* || true'
echo "Copying files, this can take some time"
# We are not using tar -r because this scans the whole archive and takes forever for big ones
ssh g5k 'find . -maxdepth 1 -type l ! -name grenoble -exec cp -r {}/exp-data {}/exp-logs {}/espec-times grenoble \; 2>&1 | grep -v "cannot stat"' &
PID=$!
while kill -0 $PID > /dev/null 2>&1; do
work_in_progress
done
echo "Creating archive, this can take some time"
ssh grenoble.g5k 'tar -czf g5k.tar.gz exp-logs exp-data espec-times' &
PID=$!
while kill -0 $PID > /dev/null 2>&1; do
work_in_progress
done
echo "Copying archive to local machine"
scp grenoble.g5k:g5k.tar.gz ${G5K_BACKUP}
test -d ${BACKUP_DIR} || mkdir ${BACKUP_DIR}
echo "Creating Backup Tar ${BACKUP_DIR}/${BACKUP_NAME}"
tar -cvf ${BACKUP_DIR}/${BACKUP_NAME} ${G5K_BACKUP} ${2} &
PID=$!
while kill -0 $PID > /dev/null 2>&1; do
work_in_progress
done
rm ${G5K_BACKUP} ${2}
}
function delete() {
echo "Cleaning up g5k storage"
ssh g5k 'rm grenoble/g5k.tar.gz || true && $(find . -type l -exec rm -rf {}/exp-logs/ {}/exp-data {}/espec-times \;)' &
PID=$!
while kill -0 $PID > /dev/null 2>&1; do
work_in_progress
done
}
echo "
Make sure that:
- Your ssh key to access the grid is added to the ssh-agent with 'ssh-add <KEY>'
- The config to access the grid is setup as described in:
https://www.grid5000.fr/w/SSH#Using_SSH_ProxyCommand_feature_to_ease_the_access_to_hosts_inside_Grid.275000
"
if [[ $(ssh-add -L) =~ "The agent has no" ]]; then
echo "No ssh key added, aborting"
exit 1
fi
if ! ssh -q -o BatchMode=yes g5k 'exit'; then
echo "SSH connection does not work, please check your configuration"
exit 1
fi
case $1 in
-b|--backup)
backup $2 $3
;;
-d|--delete)
delete
;;
*)
echo "Usage: ./persist.sh <OPTIONS>"
echo ""
echo "OPTIONS:"
echo " -b | --backup <experiment-name> <addtitional-plot-archive>"
echo " backup exp-data, exp-logs and espec-times from g5k NFS"
echo " -d | --delete"
echo " delete exp-data, exp-logs and espec-times from g5k NFS"
;;
esac
|
