Backup and restore DB guide
This document is a guideline about Backup and restore DB guide a Public Representative (“P-Rep”) node on the MainNet using a docker. P-Reps are the consensus nodes that produce, verify blocks and participate in network policy decisions on the ICON Network.

Intended Audience

We recommend all P-Rep candidates to go through this guideline.

How to backup DB

Recovering with a backed-up snapshot dramatically reduces block synchronization time. The backup must be proceeded with the node suspended.
Our backup policy is as follows.
1
backup archive cycle
2
Local disk : 1 week
3
AWS S3 : 15 days ( deleted every day night 00:00)
  • backup procedure
  • Stop Node
    • The backup must be proceeded with the node suspended.
  • To create compressed archive files of data directory and Upload to AWS S3
    • ./data/loopchain/.score_data
    • ./data/loopchain/.storage
  • Start Node

Backup script

1
#!/bin/bash
2
#
3
# ICON leveldb Cold-Backup script
4
# - requirement
5
# . awscli pkg (yum install -y awscli)
6
# . AWS access_key & secret_access_key
7
#
8
9
10
SERVICE=PRep
11
BH=`curl -s localhost:9000/api/v1/avail/peer | jq .block_height`
12
TODAY=`date "+%Y%m%d"`
13
CTIME=`date "+%H%M"`
14
FILE_NAME="${SERVICE}_BH${BH}_data-${TODAY}_${CTIME}.tar.gz"
15
PREP_DIR="/app/prep"
16
IP_DIR_NAME=`find ${PREP_DIR}/data/loopchain/.storage/*7100_icon_dex -maxdepth 1 -type d |awk -F\/ '{print $NF}'`
17
LOG_NAME="${SERVICE}_Backup_${TODAY}.log"
18
BACKUP_TARGET_DIR="${PREP_DIR}/data/loopchain"
19
BACKUP_TARGET=".score_data .storage"
20
BACKUP_DIR="${PREP_DIR}/data/Backup/${TODAY}"
21
DELETE_DIR="${PREP_DIR}/data/Backup/`date -d '1 week ago' "+%Y%m%d"`"
22
LOG_DIR="${PREP_DIR}/data/Backup/log"
23
COMPOSE_YML="docker-compose.yml"
24
PEER_STATUS=`curl -s -o /dev/null -w "%{http_code}" localhost:9000/api/v1/avail/peer`
25
S3_PATH="s3://icon-leveldb-backup/${SERVICE}"
26
S3_PATH_PERM="s3://icon-leveldb-backup/Permanent/${SERVICE}"
27
28
29
30
start_peer()
31
{
32
echo -e "[`date "+%Y-%m-%d %H:%M:%S"`] Starting Peer..."
33
cd ${PREP_DIR}
34
docker-compose -f ${COMPOSE_YML} up -d
35
}
36
37
stop_peer()
38
{
39
echo -e "[`date "+%Y-%m-%d %H:%M:%S"`] Stoping Peer..."
40
cd ${PREP_DIR}
41
docker-compose -f ${COMPOSE_YML} down
42
}
43
44
run_backup()
45
{
46
echo -e "[`date "+%Y-%m-%d %H:%M:%S"`] Start Backup"
47
cd ${BACKUP_TARGET_DIR}
48
tar -zcf ${FILE_NAME} \
49
--xform="s|.storage/${IP_DIR_NAME}|.storage/db_CHANGEIP:7100_icon_dex|" ${BACKUP_TARGET}
50
if [ ! -d ${BACKUP_DIR} ]; then
51
mkdir -p ${BACKUP_DIR}
52
fi
53
mv ${FILE_NAME} ${BACKUP_DIR}
54
cd ${BACKUP_DIR}
55
aws s3 cp ${BACKUP_DIR}/${FILE_NAME} ${S3_PATH}/${TODAY}/${FILE_NAME}
56
aws s3 ls ${S3_PATH} --recursive | grep -v backup_list | sort -r | awk -F\/ '{print $2"/"$3}' > ${BACKUP_DIR}/backup_list
57
aws s3 cp ${BACKUP_DIR}/backup_list ${S3_PATH}/backup_list
58
echo -e "[`date "+%Y-%m-%d %H:%M:%S"`] End Backup"
59
}
60
61
run_main()
62
{
63
echo -e "[`date "+%Y-%m-%d %H:%M:%S"`] Start Script"
64
if [ $PEER_STATUS == 000 ]; then
65
echo -e "\nWarning!! Service is not running!!"
66
echo -e "Backup Skip. Next schedule running backup after sync...\n"
67
stop_peer
68
start_peer
69
else
70
echo " - Block Height is [ ${BH} ] "
71
stop_peer
72
sleep 10
73
run_backup
74
start_peer
75
fi
76
if [ ${CTIME} == 0000 ]; then
77
echo -e "Delete old directory : ${DELETE_DIR}"
78
rm -rf ${DELETE_DIR}
79
echo -e "[`date "+%Y-%m-%d %H:%M:%S"`] Start Permanent Backup"
80
aws s3 cp ${BACKUP_DIR}/${FILE_NAME} ${S3_PATH_PERM}/${TODAY}/${FILE_NAME}
81
echo -e "[`date "+%Y-%m-%d %H:%M:%S"`] End Permanent Backup"
82
fi
83
echo -e "[`date "+%Y-%m-%d %H:%M:%S"`] End Script"
84
echo -e "====================================================================="
85
}
86
87
if [ ! -d ${LOG_DIR} ]; then
88
mkdir -p ${LOG_DIR}
89
fi
90
91
run_main >> ${LOG_DIR}/${LOG_NAME} 2>&1

How to restore from snapshot

  • restore procedure
  • Stop Node
  • Remove Data
    • rm -rf ./data/loopchain/.s*
  • Download backup file from AWS S3
  • Unarchive backup file and change the directory names
    • ./data/loopchain/.storage/db_{YOUR_NODE_IP}:7100_icon_dex
  • Start Node

Restore script

1
#!/bin/bash
2
RED='\033[0;31m'
3
GREEN="\033[1;32m"
4
BLUE="\033[1;34m"
5
NOCOLOR="\033[0m"
6
7
PREP_DIR="/app/prep"
8
RESTORE_DIR="${PREP_DIR}/data/loopchain"
9
10
ORIGIN_IP=`curl -s -k https://ipinfo.io/ | jq -r .ip`
11
12
S3_PATH="https://s3.ap-northeast-2.amazonaws.com/icon-leveldb-backup/PRep"
13
14
## Var Check
15
if [[ $1 == "" ]]; then
16
NOW_IP=$ORIGIN_IP
17
else
18
NOW_IP=$1
19
fi
20
21
## Process Check
22
if [ 0 != `ps -ef | grep -v grep | grep -E "icon_service|loopchain" | wc -l` ]; then
23
echo -e $RED"Process running " $GREEN
24
ps -ef | grep -v grep | grep icon
25
echo -e $RED"stop Process please" $NOCOLOR
26
exit 0
27
fi
28
29
## IP Check
30
if [[ ! $NOW_IP =~ ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3} ]]
31
then
32
echo -e $RED"IP info Fault $NOCOLOR"
33
exit 0
34
fi
35
36
function trapshell {
37
echo -e "$RED STOP SHELL $NOCOLOR"
38
exit 0;
39
}
40
41
function package_install {
42
echo -e "$GREEN #### PHASE 1 #### \n Package Install axel pigz $NOCOLOR"
43
if [ -f /etc/redhat-release ]; then
44
echo -e "$GREEN OS: centos \n Package cmd: yum install -y axel pigz$NOCOLOR"
45
yum install -y axel pigz > /dev/null 2>&1
46
elif [ -f /etc/lsb-release ]; then
47
echo -e "$GREEN OS: ubuntu \m Package cmd: apt install -y axe pigz$NOCOLOR"
48
apt install -y axel pigz > /dev/null 2>&1
49
else
50
echo "$GREEN Not Support OS $NOCOLOR"
51
fi
52
}
53
54
function Download_Backup {
55
icon_latest=`curl -s ${S3_PATH}/backup_list | head -n 1`
56
filename=`echo $icon_latest | awk -F/ '{print $NF}'`
57
rm -rf $RESTORE_DIR
58
mkdir -p $RESTORE_DIR
59
echo -e "$GREEN \n\n\n #### PHASE 2 #### \n Down Load Backup File "$filename "\n\n\n " $BLUE & axel -a ${S3_PATH}/$icon_latest --output $RESTORE_DIR/$filename
60
}
61
62
function Restore_DB {
63
echo -e "$GREEN #### PHASE 3 #### \n Restore process "$filename "\n\n\n $NOCOLOR"
64
tar -I pigz -xf $RESTORE_DIR/$filename -C $RESTORE_DIR
65
mv $RESTORE_DIR/.storage/db_CHANGEIP\:7100_icon_dex/ $RESTORE_DIR/.storage/db_$NOW_IP:7100_icon_dex
66
}
67
68
trap trapshell 1 2 15
69
package_install
70
Download_Backup
71
Restore_DB
72
73
74
echo -e "$BLUE DONE. $NOCOLOR"

prep-node's directory structure

  • Diretory structure
    1
    {P-Rep Install Directory}
    2
    └── data
    3
    └── loopchain
    4
    ├── log
    5
    ├── .score_data
    6
    │ ├── db
    7
    │ │ ├── icon_dex
    8
    │ │ ├── iiss # IISS DB
    9
    │ │ ├── meta # iconservice metadata DB
    10
    │ │ └── rc # reward calculator DB
    11
    │ └── score # state DB
    12
    └── .storage
    13
    └── db_{IP}:7100_icon_dex # block DB
1
IISS DB - IISS DB for calculating compensation amount by account transfer to reward calculator
2
metadata DB - Internal Data Storage for ICON Service without Consensus
3
RC DB - 'RC DB' will be stored I-Score value of each account from 'reward calculator'
4
state DB - SCORE data DB
5
block DB - block data DB