diff --git a/Dockerfile b/Dockerfile index 29008b0..390445d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,48 @@ -FROM ubuntu:trusty +FROM ubuntu:xenial MAINTAINER Tutum Labs -RUN apt-get update && \ - apt-get install -y --no-install-recommends mysql-client && \ - mkdir /backup +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN mkdir -p /var/lib/mysql/ \ + && groupadd -r mysql \ + && useradd -r -g mysql -d /var/lib/mysql/ mysql \ + && chown mysql:mysql /var/lib/mysql \ + && chmod 700 /var/lib/mysql + +# add gosu for easy step-down from root +ENV GOSU_VERSION 1.7 +RUN set -x \ + && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \ + && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ + && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ + && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \ + && rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \ + && chmod +x /usr/local/bin/gosu \ + && gosu nobody true \ + && apt-get purge -y --auto-remove ca-certificates wget + +# gpg: key 5072E1F5: public key "MySQL Release Engineering " imported +RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys A4A9406876FCBD3C456770C88C718D3B5072E1F5 + +ENV MYSQL_MAJOR 5.7 +ENV MYSQL_VERSION 5.7.17-1ubuntu16.04 + +RUN echo "deb http://repo.mysql.com/apt/ubuntu/ xenial mysql-${MYSQL_MAJOR}" > /etc/apt/sources.list.d/mysql.list + +RUN apt-get update \ + && apt-get install -y \ + mysql-client="${MYSQL_VERSION}" \ + cron \ + openssh-client \ + python-paramiko \ + python-pexpect \ + duplicity \ + python \ + && rm -rf /var/lib/apt/lists/* + +# timezone +ENV TIMEZONE="Etc/UTC" ENV CRON_TIME="0 0 * * *" \ MYSQL_DB="--all-databases" diff --git a/README.md b/README.md index bf9e75f..2dceeda 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ This image runs mysqldump to backup data using cronjob to folder `/backup` --env MYSQL_PORT=27017 \ --env MYSQL_USER=admin \ --env MYSQL_PASS=password \ - --volume host.folder:/backup + --volume host.folder:/backup \ + --name tutum-backup \ tutum/mysql-backup Moreover, if you link `tutum/mysql-backup` to a mysql container(e.g. `tutum/mysql`) with an alias named mysql, this image will try to auto load the `host`, `port`, `user`, `pass` if possible. @@ -19,6 +20,7 @@ Moreover, if you link `tutum/mysql-backup` to a mysql container(e.g. `tutum/mysq ## Parameters + TIMEZONE e.g. Europe/Moscow MYSQL_HOST the host/ip of your mysql database MYSQL_PORT the port number of your mysql database MYSQL_USER the username of your mysql database @@ -30,12 +32,95 @@ Moreover, if you link `tutum/mysql-backup` to a mysql container(e.g. `tutum/mysq INIT_BACKUP if set, create a backup when the container starts INIT_RESTORE_LATEST if set, restores latest backup +## Duplicity over sftp parameters + + SFTP_USER user to connect over sftp + SFTP_HOST host to connect to by sftp + SFTP_PORT port to connect to by sftp + SFTP_DIR remote directory to place files over sftp + DUPLICITY_EXTRA_OPTS usefull value: --full-if-older-than 1M --allow-source-mismatch + DUPLICITY_ENCRYPT_PASSPHRASE the encryption passphrase. keep it in a secret + DUPLICITY_SCHEME the scheme for duplicity to work with backup server. the default value is sftp. pexpect+sftp is also supported. + +## Usage with duplicity over sftp + + mkdir -p /root/.cache/duplicity + docker run -d \ + --env MYSQL_HOST=mysql.host \ + --env MYSQL_PORT=27017 \ + --env MYSQL_USER=admin \ + --env MYSQL_PASS=password \ + --env MYSQL_DB=mydatabase \ + --env EXTRA_OPTS=--skip-lock-tables --single-transaction --flush-logs --hex-blob --master-data=2 \ + --env CRON_TIME=0 6 * * * \ + --env INIT_BACKUP=1 \ + --env MAX_BACKUPS=30 \ + --env TIMEZONE=Europe/Moscow \ + --env SFTP_USER=username \ + --env SFTP_HOST=12.34.56.78 \ + --env SFTP_PORT=22 \ + --env SFTP_DIR=backup/ \ + --env DUPLICITY_EXTRA_OPTS=--full-if-older-than 1M --allow-source-mismatch \ + --env DUPLICITY_ENCRYPT_PASSPHRASE=12345676543212345676543234567654 \ + --volume /backup:/backup \ + --volume /restore:/restore \ + --volume /root/.ssh:/root/.ssh \ + --volume /root/.cache/duplicity:/root/.cache/duplicity \ + --name tutum-backup \ + tutum/mysql-backup + +You need to connect to the backup server at least once from your host system in order to have a valid record for it in the known_hosts file. + +On your host machine run this command and press Enter for all questions: + + ssh-keygen + +It would generate `/root/.ssh/id_rsa` and `/root/.ssh/id_rsa.pub` files. + +Then copy this pub key to the backup server. Be sure to allow password access first. + + cd /root/.ssh + ssh-copy-id -p 22 -i id_rsa.pub username@12.34.56.78 + +> Not forget to disallow password authentication on the backup server after pub key copying. + +It would ask for your username's password. if this command complete, try to connect: + + sftp -P 22 username@12.34.56.78 + +If this command succeeds and have not asked you for password, then you can be sure this image would function too. + +Use this [Guide](www.jscape.com/blog/setting-up-sftp-public-key-authentication-caommand-line) if you need more details about sftp configuration. + ## Restore from a backup See the list of backups, you can run: docker exec tutum-backup ls /backup + +> Note that `tutum-backup` is a docker container name assigned previously with `--name` option. To restore database from a certain backup, simply run: docker exec tutum-backup /restore.sh /backup/2015.08.06.171901 + +## Restore from a backup if using duplicity + +To restore database from a yesterday backup, simply run: + + docker exec tutum-backup /restore.sh 1D + +If you want restore only sql dump file without replacing actual database state, do it with: + + docker exec tutum-backup /restore-file-only.sh 1D + +if you have used same options for backup container startup as in the example above, +after executing it you would find file named /restore/mydatabase-1D.sql restored. + +Instead of 1D you can use 1h for hours, 15m for minutes, 1W for weeks or 1Y for years. +You can use an exact date in a format like YYYY/MM/DD. See the full list of formats there: +[Duplicity Time Formats](http://duplicity.nongnu.org/duplicity.1.html#sect8) + +If you have used your backup server for backup from several servers, e.g. `master` and `slave`, and defined the environment variable `SFTP_DIR` as `backup/1d` on `master` and to `backup/15m` on `slave`, you can use this comand to restore master backup on a slave: + + docker exec tutum-backup /bin/bash -c "export SFTP_DIR=backup/1d && export MYSQL_USER=root && export MYSQL_PASS=123456 && /restore.sh 1D" diff --git a/run.sh b/run.sh index 1116ef3..8be3b6c 100755 --- a/run.sh +++ b/run.sh @@ -16,32 +16,73 @@ MYSQL_PASS=${MYSQL_PASS:-${MYSQL_ENV_MYSQL_PASS}} [ -z "${MYSQL_USER}" ] && { echo "=> MYSQL_USER cannot be empty" && exit 1; } [ -z "${MYSQL_PASS}" ] && { echo "=> MYSQL_PASS cannot be empty" && exit 1; } -BACKUP_CMD="mysqldump -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASS} ${EXTRA_OPTS} ${MYSQL_DB} > /backup/"'${BACKUP_NAME}' +echo ${TIMEZONE} > /etc/timezone +ln -sf /usr/share/zoneinfo/${TIMEZONE} /etc/localtime echo "=> Creating backup script" rm -f /backup.sh cat <> /backup.sh #!/bin/bash + +# Setting the pass phrase to encrypt the backup files. +export PASSPHRASE=$DUPLICITY_ENCRYPT_PASSPHRASE + MAX_BACKUPS=${MAX_BACKUPS} -BACKUP_NAME=\$(date +\%Y.\%m.\%d.\%H\%M\%S).sql +BACKUP_NAME_NOEXT=\$(date +\%Y.\%m.\%d.\%H\%M\%S) +BACKUP_GZ_NAME=\${BACKUP_NAME_NOEXT}.gz + +SFTP_USER=${SFTP_USER} +SFTP_HOST=${SFTP_HOST} +SFTP_PORT=${SFTP_PORT} +SFTP_DIR="${SFTP_DIR}" +DUPLICITY_EXTRA_OPTS="${DUPLICITY_EXTRA_OPTS}" +DUPLICITY_SCHEME="${DUPLICITY_SCHEME:-sftp}" + +MYSQL_HOST=${MYSQL_HOST} +MYSQL_PORT=${MYSQL_PORT} +MYSQL_USER=${MYSQL_USER} +MYSQL_PASS="${MYSQL_PASS}" +MYSQL_DB=${MYSQL_DB} +EXTRA_OPTS="${EXTRA_OPTS}" -echo "=> Backup started: \${BACKUP_NAME}" -if ${BACKUP_CMD} ;then - echo " Backup succeeded" +if [[ ! -z \${SFTP_USER} && ! -z \${SFTP_HOST} && ! -z \${SFTP_DIR} ]]; then + echo "=> Backup started: \${MYSQL_DB}.sql" + + # using pexpect+sftp because of a bug in paramiko backend. + # @see https://lists.gnu.org/archive/html/duplicity-talk/2016-10/msg00010.html + if flock -x -n /root/.cache/duplicity/backup.lock -c "/usr/local/bin/gosu mysql mysqldump -h\${MYSQL_HOST} -P\${MYSQL_PORT} -u\${MYSQL_USER} -p\${MYSQL_PASS} \${EXTRA_OPTS} \${MYSQL_DB} > /backup/\${MYSQL_DB}.sql && duplicity --ssh-options=\"-oProtocol=2 -oIdentityFile=/root/.ssh/id_rsa\" \${DUPLICITY_EXTRA_OPTS} /backup \${DUPLICITY_SCHEME}://\${SFTP_USER}@\${SFTP_HOST}:\${SFTP_PORT}/\${SFTP_DIR}" ;then + echo " Backup succeeded" + else + echo " Backup failed" + fi + + if [ -n "\${MAX_BACKUPS}" ]; then + if flock -x -n /root/.cache/duplicity/backup.lock -c "duplicity remove-older-than \${MAX_BACKUPS} --force --ssh-options=\"-oProtocol=2 -oIdentityFile=/root/.ssh/id_rsa\" \${DUPLICITY_SCHEME}://\${SFTP_USER}@\${SFTP_HOST}:\${SFTP_PORT}/\${SFTP_DIR}" ;then + echo " Backup succeeded" + else + echo " Backup failed" + fi + fi else - echo " Backup failed" - rm -rf /backup/\${BACKUP_NAME} -fi + echo "=> Backup started: \${BACKUP_GZ_NAME}" -if [ -n "\${MAX_BACKUPS}" ]; then - while [ \$(ls /backup -N1 | wc -l) -gt \${MAX_BACKUPS} ]; - do - BACKUP_TO_BE_DELETED=\$(ls /backup -N1 | sort | head -n 1) - echo " Backup \${BACKUP_TO_BE_DELETED} is deleted" - rm -rf /backup/\${BACKUP_TO_BE_DELETED} - done + if flock -x -n /root/.cache/duplicity/backup.lock -c "exec /usr/local/bin/gosu mysql mysqldump -h\${MYSQL_HOST} -P\${MYSQL_PORT} -u\${MYSQL_USER} -p\${MYSQL_PASS} \${EXTRA_OPTS} \${MYSQL_DB} | gzip -c -9 > /backup/\${BACKUP_GZ_NAME}" ;then + echo " Backup succeeded" + else + echo " Backup failed" + fi + + if [ -n "\${MAX_BACKUPS}" ]; then + while [ \$(ls /backup -N1 | wc -l) -gt \${MAX_BACKUPS} ]; + do + BACKUP_TO_BE_DELETED=\$(ls /backup -N1 | sort | head -n 1) + echo " Backup \${BACKUP_TO_BE_DELETED} is deleted" + rm -rf /backup/\${BACKUP_TO_BE_DELETED} + done + fi fi + echo "=> Backup done" EOF chmod +x /backup.sh @@ -50,16 +91,77 @@ echo "=> Creating restore script" rm -f /restore.sh cat <> /restore.sh #!/bin/bash -echo "=> Restore database from \$1" -if mysql -h${MYSQL_HOST} -P${MYSQL_PORT} -u${MYSQL_USER} -p${MYSQL_PASS} < \$1 ;then - echo " Restore succeeded" + +# Setting the pass phrase to encrypt the backup files. +export PASSPHRASE=\$DUPLICITY_ENCRYPT_PASSPHRASE + +SFTP_USER=${SFTP_USER} +SFTP_HOST=${SFTP_HOST} +SFTP_PORT=${SFTP_PORT} +SFTP_DIR="${SFTP_DIR}" +DUPLICITY_SCHEME="${DUPLICITY_SCHEME:-sftp}" + +MYSQL_HOST=${MYSQL_HOST} +MYSQL_PORT=${MYSQL_PORT} +MYSQL_USER=${MYSQL_USER} +MYSQL_PASS="${MYSQL_PASS}" +MYSQL_DB=${MYSQL_DB} + +if [[ ! -z \${SFTP_USER} && ! -z \${SFTP_HOST} && ! -z \${SFTP_DIR} ]]; then + # using pexpect+sftp because of a bug in paramiko backend. + # @see https://lists.gnu.org/archive/html/duplicity-talk/2016-10/msg00010.html + if duplicity --allow-source-mismatch --ssh-options="-oProtocol=2 -oIdentityFile=/root/.ssh/id_rsa" -t \${1} --file-to-restore \${MYSQL_DB}.sql \${DUPLICITY_SCHEME}://\${SFTP_USER}@\${SFTP_HOST}:\${SFTP_PORT}/\${SFTP_DIR} /restore/\${MYSQL_DB}-\${1}.sql && gosu mysql mysql -h\${MYSQL_HOST} -P\${MYSQL_PORT} -u\${MYSQL_USER} -p\${MYSQL_PASS} < /restore/\${MYSQL_DB}-\${1}.sql ;then + echo " Restore succeeded" + else + echo " Restore failed" + fi else - echo " Restore failed" + echo "=> Restore database from \$1" + if gunzip -c \$1 | exec gosu mysql mysql -h\${MYSQL_HOST} -P\${MYSQL_PORT} -u\${MYSQL_USER} -p\${MYSQL_PASS} ;then + echo " Restore succeeded" + else + echo " Restore failed" + fi fi echo "=> Done" EOF chmod +x /restore.sh +rm -f /restore-file-only.sh +cat <> /restore-file-only.sh +#!/bin/bash + +# Setting the pass phrase to encrypt the backup files. +export PASSPHRASE=\$DUPLICITY_ENCRYPT_PASSPHRASE + +SFTP_USER=${SFTP_USER} +SFTP_HOST=${SFTP_HOST} +SFTP_PORT=${SFTP_PORT} +SFTP_DIR="${SFTP_DIR}" +DUPLICITY_SCHEME="${DUPLICITY_SCHEME:-sftp}" + +MYSQL_DB=${MYSQL_DB} + +if [[ ! -z \${SFTP_USER} && ! -z \${SFTP_HOST} && ! -z \${SFTP_DIR} ]]; then + # using pexpect+sftp because of a bug in paramiko backend. + # @see https://lists.gnu.org/archive/html/duplicity-talk/2016-10/msg00010.html + if duplicity --allow-source-mismatch --ssh-options="-oProtocol=2 -oIdentityFile=/root/.ssh/id_rsa" -t \${1} --file-to-restore \${MYSQL_DB}.sql ${DUPLICITY_SCHEME}://\${SFTP_USER}@\${SFTP_HOST}:\${SFTP_PORT}/\${SFTP_DIR} /restore/\${MYSQL_DB}-\${1}.sql ;then + echo " Restore succeeded" + else + echo " Restore failed" + fi +else + echo "=> Restore database from \$1" + if gunzip -c \$1 > /restore/\${1}.sql ;then + echo " Restore succeeded" + else + echo " Restore failed" + fi +fi +echo "=> Done" +EOF +chmod +x /restore-file-only.sh + touch /mysql_backup.log tail -F /mysql_backup.log &