From 18b026e1ddbf6b0e02bbe6cc25a2d965c1dcb9fd Mon Sep 17 00:00:00 2001 From: Neil Dunbar Date: Thu, 14 Nov 2013 16:45:57 +0000 Subject: [PATCH] Initial Commit --- Dockerfile | 46 +++ README.md | 175 +++++++++ mariadb-setrootpassword | 114 ++++++ mariadb-start | 66 ++++ my-cluster.cnf | 82 +++++ my-init.cnf | 71 ++++ wsrep_sst_common | 127 +++++++ wsrep_sst_xtrabackup-v2 | 767 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 1448 insertions(+) create mode 100644 Dockerfile create mode 100644 README.md create mode 100755 mariadb-setrootpassword create mode 100755 mariadb-start create mode 100644 my-cluster.cnf create mode 100644 my-init.cnf create mode 100755 wsrep_sst_common create mode 100755 wsrep_sst_xtrabackup-v2 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..45df405 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +from ubuntu:12.04 + +# Configure apt +run echo 'deb http://us.archive.ubuntu.com/ubuntu/ precise universe' >> /etc/apt/sources.list +run apt-get -y update +run apt-get -y install python-software-properties +run apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xcbcb082a1bb943db +run add-apt-repository 'deb http://ftp.osuosl.org/pub/mariadb/repo/5.5/ubuntu precise main' +run apt-key adv --recv-keys --keyserver keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A + +# Make apt and MariaDB happy with the docker environment +run echo "#!/bin/sh\nexit 101" >/usr/sbin/policy-rc.d +run chmod +x /usr/sbin/policy-rc.d +run cat /proc/mounts >/etc/mtab + +# Install MariaDB +run apt-get -y update +run apt-get -y install +run LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get install -y iproute mariadb-galera-server galera rsync netcat-openbsd socat pv + +# this is for testing - can be commented out later +run LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get install -y iputils-ping net-tools + +run add-apt-repository 'deb http://repo.percona.com/apt precise main' +run apt-get -y update +run LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get install -y percona-xtrabackup + +# add in extra wsrep scripts +add wsrep_sst_common /usr/bin/wsrep_sst_common +add wsrep_sst_xtrabackup-v2 /usr/bin/wsrep_sst_xtrabackup-v2 + +# Clean up +run rm /usr/sbin/policy-rc.d +run rm -r /var/lib/mysql + +# Add config(s) - standalong and cluster mode +add ./my-cluster.cnf /etc/mysql/my-cluster.cnf +add ./my-init.cnf /etc/mysql/my-init.cnf + +expose 3306 4567 + +add ./mariadb-setrootpassword /usr/bin/mariadb-setrootpassword +add ./mariadb-start /usr/bin/mariadb-start +cmd ["/usr/bin/mariadb-start"] + +# vim:ts=8:noet: diff --git a/README.md b/README.md new file mode 100644 index 0000000..d5f2e72 --- /dev/null +++ b/README.md @@ -0,0 +1,175 @@ +MariaDB 5.5 Galera +================== + +A reasonably simple Ubuntu 12.04 LTS container with the pieces to form +a MariaDB 5.5 Galera cluster. It's based on Nick Stenning's MariaDB +5.5 container for MariaDB 5.5, and adds in default support for X.509 +based administrative authentication. + +(Repeating the warning from Nick S) + +**NB**: Please be aware that by default docker will make the MariaDB +port accessible from anywhere if the host firewall is unconfigured. It +also exposes the Galera wsrep port (by default 4567) globally. + +The root password for the first node (and therefore the entire +cluster) is randomly generated. This password can only be used to +contact the server along the unix domain socket (ie, the default +localhost connection). + +The container needs to import two volumes (one read/write, the other +read only). The first volume is the data volume, where the actual +MySQL data is stored. + +The second volume (read-only) is the SSL container, which should +contain 3 files: + +* a CA certificate (this can be a real CA certificate, or just a + simple test one, generated via something like TinyCA. This CA + file should be called `ca.pem`. [^1] +* an SSL private key. This should be set to ownership mode 0600 (or + even 0400). The file needs to be called `server-key.pem`. +* an SSL server certificate. The CN component of the subject name on + this certificate should be the DNS name of the host which will be + the end point for the database service. Note that you can have + multiple host names by using the subjectAltName X.509 v3 extension + on the certificate. This file needs to be called `server-cert.pem`. + +The data volume must be mounted at the container directory +`/var/lib/mysql`. + +The SSL volume must be mounted at the container directory +`/etc/ssl/mysql`. + +Within the SSL volume there should also be a directory called `root` +which will contain a set of *client* certificates with names like +`joe-root.pem`, `amy-root.pem`, `superman.pem` and so on. These names +will be registered as root capable users from any network host, with +no passwords, but only available using the SSL client certificates. +[^2] + +The remote root access is derived from a set of X.509 certificates +which will be used via SSL to authenticate the user, so that passwords +are not passed in via environmental variables. The root password is +also encrypted via those keys and stored on the (exported) data volume +which the container needs. + +Decrypting the Root Password +---------------------------- + +If you need to recover the root password, and you are the holder of +one of the root keys in /etc/ssl/mysql/root, then execute + +$ cat _/path/to/mysql-dir/_rootpw.pem | openssl smime -decrypt -inkey +_path/to/private-key.pem_ + +e.g. + +cat /data/mysql-node-1/rootpw.pem | openssl smime -decrypt -inkey +~/ssl-keys/joe-root-key.pem + + +Example Usage +------------- + +This sets up a 3 node cluster on 3 different servers, all using +default ports. + +We assume that /data/mysql holds the (host, not container) directory +where the MySQL database files need to go, and /data/mysql-ssl +contains the relevant keys, certificates and CA certificates. For the +first node standup, /data/mysql should be empty. + +# Boot the first node in standalone mode # + +DBID=$(sudo docker run -d -v /data/mysql:/var/lib/mysql \ + -v /data/mysql-ssl:/etc/ssl/mysql \ + -p 3306:3306 -e CLUSTER=BOOT ndunbar/mariadb55 \ + mariadb-start) + +(Note: no need to expose ports 4567 and 4444 - no-one is talking to +anyone just yet) + +This should fire up a single MySQL server with the appropriate remote +root user permissions (using SSL client certificates only). Check the +contents of `/data/mysql/mysql.error.log` for whether things worked OK +or not. + +Assuming it went well, continue. + +# Stop the standalone mode # + +sudo docker stop $DBID + +This should (gracefully) stop the standalone mode MariaDB node + +# Start the node in Primary Component mode # + +To begin the cluster, you need a single node as a catch-all donor. + +DBID=$(sudo docker run -d /data/mysql:/var/lib/mysql \ + -v /data/mysql-ssl:/etc/ssl/mysql \ + -p 3306:3306 -p 4567:4567 -p 4444:4444 \ + -e CLUSTER=INIT \ + -e NODE_ADDR=my.host.name ndunbar/mariadb55 \ + mariadb-start) + +This should restart the server (with data intact) with the ability to +use it to populate new nodes to fill out the cluster. Node that the +primary component has a node number of 1 (this is fixed by scripting). + +Note that ports 4567 and 4444 are open. 4567 is the Galera clustering +service, and 4444 is the service by which snapshots of the database +are transferred across to bring a new node into sync. The current +container implementation uses Xtrabackup, encrypting the transfers via +SSL between nodes. Xtrabackup has the nice property of not blocking +the donor server (well, it does, but only for a very short time), +which means you can continue to write into the donor node while other +nodes are coming online. + +# Start the other nodes in the cluster + +For each extra node that you need to turn up, set up the `/data/mysql` +and `/data/mysql-ssl` directories as per the primary node, and execute + +DBID=$(sudo docker run -d /data/mysql:/var/lib/mysql \ + -v /data/mysql-ssl:/etc/ssl/mysql \ + -p 3306:3306 -p 4567:4567 -p 4444:4444 \ + -e CLUSTER= my.host.name,his.host.name,her.host.name \ + -e NODE= \ + -e NODE_ADDR=my.host.name ndunbar/mariadb55 \ + mariadb-start) + +where `` is some integer above 1, ideally sequential. The +`his.host.name`, `her.host.name` etc. should be the other nodes in the +cluster. You don't need to list all the nodes in the cluster - the +service will discover the other active nodes upon attempting to join +the cluster - if the list includes the host name of the node on which +the server is running, it will be excluded from the list of nodes to +be interrogated for cluster manifest. + +Three nodes is the least number needed to avoid split brain +scenarios. + +# (Optional) Restart the primary node as a normal cluster node # + +Once the minimum number of synced nodes the cluster reaches 3, you can +stop and restart the primary component as just another node in the +cluster. + +(On the server running the primary node) + +sudo docker stop $DBID + +DBID=$(sudo docker run -d /data/mysql:/var/lib/mysql \ + -v /data/mysql-ssl:/etc/ssl/mysql \ + -p 3306:3306 -p 4567:4567 -p 4444:4444 \ + -e CLUSTER= my.host.name,his.host.name,her.host.name \ + -e NODE=1 \ + -e NODE_ADDR=my.host.name ndunbar/mariadb55 \ + mariadb-start) + +[^1]: http://tinyca.sm-zone.net + +[^2]: Note that the root user certificates _must_ be signed by the +same CA as the one signing the server certificate. diff --git a/mariadb-setrootpassword b/mariadb-setrootpassword new file mode 100755 index 0000000..621f481 --- /dev/null +++ b/mariadb-setrootpassword @@ -0,0 +1,114 @@ +#!/bin/bash + +set -u + +MSA=/usr/bin/mysqladmin +# generate long random password +MARIADB_ROOT_PW=$(openssl rand -base64 32) + + +# make sure ownership of data dir is OK +chown -R mysql:mysql /var/lib/mysql +/usr/bin/mysqld_safe & + + + +sleep 5 # wait for mysqld_safe to rev up, and check for port 3306 +port_open=0 + +while [ "$port_open" -eq 0 ]; do + /bin/nc -z -w 5 127.0.0.1 3306 + if [ $? -ne 0 ]; then + echo "Sleeping waiting for port 3306 to open: result " $? + sleep 1 + else + echo "Port 3306 is open" + port_open=1 + fi +done + +# Secure the installation +done=0 +count=0 +maxtries=10 +while [ $done -eq 0 ]; do + ${MSA} -u root password ${MARIADB_ROOT_PW} + if [ $? -ne 0 ]; then + count=$((${count} + 1)) + if [ $count -gt $maxtries ]; then + echo "Maximum tries at setting password exceeded. Giving up" + exit 1 + else + echo "Root password set failed. Sleeping, then retrying" + sleep 1 + fi + else + echo "Root Password set successfully" + done=1 + fi +done + +# this code mimics the secure install script, which was originally +# scripted via expect. I found that unreliable, hence this section + +# drop test database +echo "Dropping test DB" +DROP="DROP DATABASE IF EXISTS test" +echo "$DROP" | mysql -u root --password="$MARIADB_ROOT_PW" mysql + +echo "Cleaning test db privs" +# remove db privs for test +DELETE="DELETE FROM mysql.db Where Db='test' OR Db='test\\_%'" +echo "$DELETE" | mysql -u root --password="$MARIADB_ROOT_PW" mysql + +echo "Deleting anon db users" +# remove anon users +DELETE="DELETE FROM mysql.user WHERE User=''" +echo "$DELETE" | mysql -u root --password="$MARIADB_ROOT_PW" mysql + +echo "create mysql user" +# create mysql@localhost user for xtrabackup +CUSER="CREATE USER 'mysql'@'localhost'" +echo "$CUSER" | mysql -u root --password="$MARIADB_ROOT_PW" mysql + +# now set the root passwords given the certificates available +ROOTCERTS=/etc/ssl/mysql/root +if [ -d ${ROOTCERTS} ]; then + clist=$(find ${ROOTCERTS} -type f -a -name \*.pem -print) + + if [ -z "${clist}" ]; then + echo "No certificates available to encrypt root pw" >&2 + exit 1 + fi + + # dumping encrypted root password to disk - make sure that + # only owner can read/write it + oldu=$(umask) + umask 0077 + echo -n $MARIADB_ROOT_PW | openssl smime -encrypt -aes256 \ + -out /var/lib/mysql/rootpw.pem \ + ${clist} + umask ${oldu} + + for c in ${clist}; do + filename=$(basename "${c}") + username=${filename%.*} + # extract subject and issuer + subject=$(openssl x509 -noout -subject -in ${c} | sed -e "s/^subject= //ig") + printf -v qsubject "%q" "${subject}" + issuer=$(openssl x509 -noout -issuer -in ${c} | sed -e "s/^issuer= //ig" -e "") + printf -v qissuer "%q" "${issuer}" + CUSER="CREATE USER '${username}'@'%'" + echo "${CUSER}" | mysql -u root --password="$MARIADB_ROOT_PW" mysql + GRANT="GRANT ALL PRIVILEGES ON *.* TO '${username}'@'%' \ + REQUIRE SUBJECT '${qsubject}' AND \ + ISSUER '${qissuer}' \ + WITH GRANT OPTION" + echo "${GRANT}" | mysql -u root --password="$MARIADB_ROOT_PW" mysql + done +fi +echo "FLUSH PRIVILEGES" | mysql -u root --password="$MARIADB_ROOT_PW" mysql + +echo "Shutting down MySQL server" +${MSA} -uroot -p${MARIADB_ROOT_PW} shutdown +echo "---> MariaDB installation secured with root password" >&2 diff --git a/mariadb-start b/mariadb-start new file mode 100755 index 0000000..b60b9b9 --- /dev/null +++ b/mariadb-start @@ -0,0 +1,66 @@ +#!/bin/sh + +set -eu + +status () { + echo "---> ${@}" >&2 +} + +if [ -z ${CLUSTER+x} ]; then + echo "CLUSTER variable must be BOOT, INIT or cluster node list" + exit 1 +elif [ ${CLUSTER} = "BOOT" ]; then + ln -sf /etc/mysql/my-init.cnf /etc/mysql/my.cnf +else + ln -sf /etc/mysql/my-cluster.cnf /etc/mysql/my.cnf +fi + +mv /usr/bin/xtrabackup /usr/bin/xtrabackup_51 +ln -sf /usr/bin/xtrabackup_55 /usr/bin/xtrabackup + +if [ -d /etc/ssl/mysql ]; then + chown mysql:mysql /etc/ssl/mysql/*.pem +fi + +# create link for less flexible scripts + +ln -sf /var/lib/mysql/mysql.sock /var/run/mysqld/mysqld.sock + +if [ -z ${CLUSTER+x} ]; then + echo "CLUSTER variable must be BOOT, INIT or cluster node list" + exit 1 +elif [ ${CLUSTER} = "BOOT" ]; then + if [ ! -e /var/lib/mysql/bootstrapped ]; then + status "Bootstrapping MariaDB installation..." + + status "Initializing MariaDB root directory at /var/lib/mysql" + mysql_install_db + + status "Setting MariaDB root password" + /usr/bin/mariadb-setrootpassword + touch /var/lib/mysql/bootstrapped + else + status "Starting from already-bootstrapped MariaDB installation" + fi + + exec /usr/bin/mysqld_safe --log-error=/var/lib/mysql/mysql.error.log +elif [ ${CLUSTER} = "INIT" ]; then + NODE=1 # force this + # grab the ip address of eth0 if NODE_ADDR is not provided + NODE_ADDR=${NODE_ADDR:-$(ip -4 addr show eth0 | grep inet | sed -e 's/ */ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1)} + exec /usr/bin/mysqld_safe --wsrep_node_address="${NODE_ADDR}" \ + --wsrep_node_incoming_address="${NODE_ADDR}" \ + --wsrep_new_cluster --wsrep_cluster_address="gcomm://" \ + --log-error=/var/lib/mysql/mysql-$NODE.error.log +else + if [ -z ${NODE+x} ]; then + echo "Must define node number" >&2 + exit 1 + fi + NODE_ADDR=${NODE_ADDR:-$(ip -4 addr show eth0 | grep inet | sed -e 's/ */ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1)} + CLUSTER="gcomm://$CLUSTER" + exec /usr/bin/mysqld_safe --wsrep_node_address="${NODE_ADDR}" \ + --wsrep_node_incoming_address="${NODE_ADDR}" \ + --wsrep_cluster_address=$CLUSTER --log-error=/var/lib/mysql/mysql-$NODE.error.log +fi + diff --git a/my-cluster.cnf b/my-cluster.cnf new file mode 100644 index 0000000..d9a0ed3 --- /dev/null +++ b/my-cluster.cnf @@ -0,0 +1,82 @@ +[mysql] + +# CLIENT # +port = 3306 +socket = /var/lib/mysql/mysql.sock + +[mysqld] + +# GENERAL # +user = mysql +default-storage-engine = InnoDB +socket = /var/lib/mysql/mysql.sock +pid-file = /var/lib/mysql/mysql.pid + +# SSL # +ssl-ca = /etc/ssl/mysql/ca.pem +ssl-cert = /etc/ssl/mysql/server-cert.pem +ssl-key = /etc/ssl/mysql/server-key.pem + +# MyISAM # +key-buffer-size = 32M +myisam-recover = FORCE,BACKUP + +# SAFETY # +max-allowed-packet = 16M +max-connect-errors = 1000000 +skip-name-resolve +sql-mode = STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_AUTO_VALUE_ON_ZERO,NO_ENGINE_SUBSTITUTION,NO_ZERO_DATE,NO_ZERO_IN_DATE,ONLY_FULL_GROUP_BY +sysdate-is-now = 1 +innodb = FORCE +innodb-strict-mode = 1 +innodb-autoinc-lock-mode = 2 +innodb-doublewrite = 1 +innodb_flush_log_at_trx_commit = 2 +innodb_file_per_table = 1 + +# DATA STORAGE # +datadir = /var/lib/mysql/ + +# BINARY LOGGING # +log-bin = /var/lib/mysql/mysql-bin +expire-logs-days = 14 +sync-binlog = 1 +binlog-format = row + +# CACHES AND LIMITS # +tmp-table-size = 32M +max-heap-table-size = 32M +query-cache-type = 0 +query-cache-size = 0 +max-connections = 500 +thread-cache-size = 50 +open-files-limit = 65535 +table-definition-cache = 4096 +table-open-cache = 4096 + +# INNODB # +innodb-flush-method = O_DIRECT +innodb-log-files-in-group = 2 +innodb-log-file-size = 128M +innodb-flush-log-at-trx-commit = 1 +innodb-file-per-table = 1 +innodb-buffer-pool-size = 128M + +# LOGGING # +log-error = /var/lib/mysql/mysql-error.log +log-queries-not-using-indexes = 1 +slow-query-log = 1 +slow-query-log-file = /var/lib/mysql/mysql-slow.log + +[mariadb] +wsrep_provider = /usr/lib/galera/libgalera_smm.so +wsrep_sst_method = xtrabackup-v2 + +[sst] +streamfmt = xbstream +transferfmt = socat +progress = 1 +tca = /etc/ssl/mysql/ca.pem +tcert = /etc/ssl/mysql/server-cert.pem +tkey = /etc/ssl/mysql/server-key.pem +encrypt = 3 \ No newline at end of file diff --git a/my-init.cnf b/my-init.cnf new file mode 100644 index 0000000..02a26c2 --- /dev/null +++ b/my-init.cnf @@ -0,0 +1,71 @@ +[mysql] + +# CLIENT # +port = 3306 +socket = /var/lib/mysql/mysql.sock + +[mysqld] + +# GENERAL # +user = mysql +default-storage-engine = InnoDB +socket = /var/lib/mysql/mysql.sock +pid-file = /var/lib/mysql/mysql.pid + +# SSL # +ssl-ca = /etc/ssl/mysql/ca.pem +ssl-cert = /etc/ssl/mysql/server-cert.pem +ssl-key = /etc/ssl/mysql/server-key.pem + +# MyISAM # +key-buffer-size = 32M +myisam-recover = FORCE,BACKUP + +# SAFETY # +max-allowed-packet = 16M +max-connect-errors = 1000000 +skip-name-resolve +sql-mode = STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_AUTO_VALUE_ON_ZERO,NO_ENGINE_SUBSTITUTION,NO_ZERO_DATE,NO_ZERO_IN_DATE,ONLY_FULL_GROUP_BY +sysdate-is-now = 1 +innodb = FORCE +innodb-strict-mode = 1 +innodb-autoinc-lock-mode = 2 +innodb-doublewrite = 1 +innodb_flush_log_at_trx_commit = 2 +innodb_file_per_table = 1 + +# DATA STORAGE # +datadir = /var/lib/mysql/ + +# BINARY LOGGING # +log-bin = /var/lib/mysql/mysql-bin +expire-logs-days = 14 +sync-binlog = 1 +binlog-format = row + +# CACHES AND LIMITS # +tmp-table-size = 32M +max-heap-table-size = 32M +query-cache-type = 0 +query-cache-size = 0 +max-connections = 500 +thread-cache-size = 50 +open-files-limit = 65535 +table-definition-cache = 4096 +table-open-cache = 4096 + +# INNODB # +innodb-flush-method = O_DIRECT +innodb-log-files-in-group = 2 +innodb-log-file-size = 128M +innodb-flush-log-at-trx-commit = 1 +innodb-file-per-table = 1 +innodb-buffer-pool-size = 128M + +# LOGGING # +log-error = /var/lib/mysql/mysql-error.log +log-queries-not-using-indexes = 1 +slow-query-log = 1 +slow-query-log-file = /var/lib/mysql/mysql-slow.log + + diff --git a/wsrep_sst_common b/wsrep_sst_common new file mode 100755 index 0000000..f9a08c1 --- /dev/null +++ b/wsrep_sst_common @@ -0,0 +1,127 @@ +# Copyright (C) 2012 Codership Oy +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# This is a common command line parser to be sourced by other SST scripts + +set -u + +WSREP_SST_OPT_BYPASS=0 +WSREP_SST_OPT_DATA="" + +while [ $# -gt 0 ]; do +case "$1" in + '--address') + readonly WSREP_SST_OPT_ADDR="$2" + shift + ;; + '--auth') + WSREP_SST_OPT_AUTH="$2" + shift + ;; + '--bypass') + WSREP_SST_OPT_BYPASS=1 + ;; + '--datadir') + readonly WSREP_SST_OPT_DATA="$2" + shift + ;; + '--defaults-file') + readonly WSREP_SST_OPT_CONF="$2" + shift + ;; + '--host') + readonly WSREP_SST_OPT_HOST="$2" + shift + ;; + '--local-port') + readonly WSREP_SST_OPT_LPORT="$2" + shift + ;; + '--parent') + readonly WSREP_SST_OPT_PARENT="$2" + shift + ;; + '--password') + readonly WSREP_SST_OPT_PSWD="$2" + shift + ;; + '--port') + readonly WSREP_SST_OPT_PORT="$2" + shift + ;; + '--role') + readonly WSREP_SST_OPT_ROLE="$2" + shift + ;; + '--socket') + readonly WSREP_SST_OPT_SOCKET="$2" + shift + ;; + '--user') + readonly WSREP_SST_OPT_USER="$2" + shift + ;; + '--gtid') + readonly WSREP_SST_OPT_GTID="$2" + shift + ;; + *) # must be command + # usage + # exit 1 + ;; +esac +shift +done +readonly WSREP_SST_OPT_BYPASS + +# For Bug:1200727 +if my_print_defaults -c $WSREP_SST_OPT_CONF sst | grep -q "wsrep_sst_auth";then + if [ -z $WSREP_SST_OPT_AUTH -o $WSREP_SST_OPT_AUTH = "(null)" ];then + WSREP_SST_OPT_AUTH=$(my_print_defaults -c $WSREP_SST_OPT_CONF sst | grep -- "--wsrep_sst_auth" | cut -d= -f2) + fi +fi + +if [ -n "${WSREP_SST_OPT_DATA:-}" ] +then + SST_PROGRESS_FILE="$WSREP_SST_OPT_DATA/sst_in_progress" +else + SST_PROGRESS_FILE="" +fi + + +wsrep_log() +{ + # echo everything to stderr so that it gets into common error log + # deliberately made to look different from the rest of the log + local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)" + echo "WSREP_SST: $* ($tst)" >&2 +} + +wsrep_log_error() +{ + wsrep_log "[ERROR] $*" +} + +wsrep_log_info() +{ + wsrep_log "[INFO] $*" +} + +wsrep_cleanup_progress_file() +{ + [ -n "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null +} + diff --git a/wsrep_sst_xtrabackup-v2 b/wsrep_sst_xtrabackup-v2 new file mode 100755 index 0000000..afaf33d --- /dev/null +++ b/wsrep_sst_xtrabackup-v2 @@ -0,0 +1,767 @@ +#!/bin/bash -ue +# Copyright (C) 2013 Percona Inc +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# Documentation: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html +# Make sure to read that before proceeding! + + + + +. $(dirname $0)/wsrep_sst_common + +ealgo="" +ekey="" +ekeyfile="" +encrypt=0 +nproc=1 +ecode=0 +XTRABACKUP_PID="" +SST_PORT="" +REMOTEIP="" +tcert="" +tpem="" +tkey="" +sockopt="" +progress="" +ttime=0 +totime=0 +lsn="" +incremental=0 +ecmd="" +rlimit="" +# Initially +stagemsg="${WSREP_SST_OPT_ROLE}" +cpat="" + +sfmt="tar" +strmcmd="" +tfmt="" +tcmd="" +rebuild=0 +rebuildcmd="" +payload=0 +pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " +pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " +STATDIR="" +uextra=0 + +if which pv &>/dev/null && pv --help | grep -q FORMAT;then + pvopts+=$pvformat +fi +pcmd="pv $pvopts" +declare -a RC + +INNOBACKUPEX_BIN=innobackupex +readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ }) +DATA="${WSREP_SST_OPT_DATA}" +INFO_FILE="xtrabackup_galera_info" +IST_FILE="xtrabackup_ist" +MAGIC_FILE="${DATA}/${INFO_FILE}" + +# Setting the path for ss and ip +export PATH="/usr/sbin:/sbin:$PATH" + +timeit(){ + local stage=$1 + shift + local cmd="$@" + local x1 x2 took extcode + + if [[ $ttime -eq 1 ]];then + x1=$(date +%s) + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + x2=$(date +%s) + took=$(( x2-x1 )) + wsrep_log_info "NOTE: $stage took $took seconds" + totime=$(( totime+took )) + else + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + fi + return $extcode +} + +get_keys() +{ + if [[ $encrypt -ge 2 ]];then + return + fi + + if [[ $encrypt -eq 0 ]];then + if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then + wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " + fi + return + fi + + if [[ $sfmt == 'tar' ]];then + wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" + encrypt=0 + return + fi + + wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" + + if [[ -z $ealgo ]];then + wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" + exit 3 + fi + + if [[ -z $ekey && ! -r $ekeyfile ]];then + wsrep_log_error "FATAL: Either key or keyfile must be readable" + exit 3 + fi + + if [[ -z $ekey ]];then + ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" + else + ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey" + fi + + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + ecmd+=" -d" + fi + + stagemsg+="-XB-Encrypted" +} + +get_transfer() +{ + if [[ -z $SST_PORT ]];then + TSST_PORT=4444 + else + TSST_PORT=$SST_PORT + fi + + if [[ $tfmt == 'nc' ]];then + if [[ ! -x `which nc` ]];then + wsrep_log_error "nc(netcat) not found in path: $PATH" + exit 2 + fi + wsrep_log_info "Using netcat as streamer" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="nc -dl ${TSST_PORT}" + else + tcmd="nc ${REMOTEIP} ${TSST_PORT}" + fi + else + tfmt='socat' + wsrep_log_info "Using socat as streamer" + if [[ ! -x `which socat` ]];then + wsrep_log_error "socat not found in path: $PATH" + exit 2 + fi + + if [[ $encrypt -eq 2 || $encrypt -eq 3 ]] && ! socat -V | grep -q WITH_OPENSSL;then + wsrep_log_info "NOTE: socat is not openssl enabled, falling back to plain transfer" + encrypt=0 + fi + + if [[ $encrypt -eq 2 ]];then + wsrep_log_info "Using openssl based encryption with socat: with crt and pem" + if [[ -z $tpem || -z $tcert ]];then + wsrep_log_error "Both PEM and CRT files required" + exit 22 + fi + stagemsg+="-OpenSSL-Encrypted-2" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + wsrep_log_info "Decrypting with PEM $tpem, CA: $tcert" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,cafile=${tcert}${sockopt} stdio" + else + wsrep_log_info "Encrypting with PEM $tpem, CA: $tcert" + tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,cafile=${tcert}${sockopt}" + fi + elif [[ $encrypt -eq 3 ]];then + wsrep_log_info "Using openssl based encryption with socat: with key and crt" + if [[ -z $tpem || -z $tkey ]];then + wsrep_log_error "Both certificate and key files required" + exit 22 + fi + stagemsg+="-OpenSSL-Encrypted-3" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + wsrep_log_info "Decrypting with certificate $tpem, key $tkey" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,key=${tkey},verify=0${sockopt} stdio" + else + wsrep_log_info "Encrypting with certificate $tpem, key $tkey" + tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,key=${tkey},verify=0${sockopt}" + fi + + else + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" + else + tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}" + fi + fi + fi + +} + +parse_cnf() +{ + local group=$1 + local var=$2 + reval=$(my_print_defaults -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2-) + if [[ -z $reval ]];then + [[ -n $3 ]] && reval=$3 + fi + echo $reval +} + +get_footprint() +{ + pushd $WSREP_SST_OPT_DATA 1>/dev/null + payload=$(du --block-size=1 -c **/*.ibd **/*.MYI **/*.MYI ibdata1 | awk 'END { print $1 }') + if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then + # QuickLZ has around 50% compression ratio + # When compression/compaction used, the progress is only an approximate. + payload=$(( payload*1/2 )) + fi + popd 1>/dev/null + pcmd+=" -s $payload" + adjust_progress +} + +adjust_progress() +{ + if [[ -n $progress && $progress != '1' ]];then + if [[ -e $progress ]];then + pcmd+=" 2>>$progress" + else + pcmd+=" 2>$progress" + fi + elif [[ -z $progress && -n $rlimit ]];then + # When rlimit is non-zero + pcmd="pv -q" + fi + + if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then + wsrep_log_info "Rate-limiting SST to $rlimit" + pcmd+=" -L \$rlimit" + fi +} + +read_cnf() +{ + sfmt=$(parse_cnf sst streamfmt "xbstream") + tfmt=$(parse_cnf sst transferfmt "socat") + tcert=$(parse_cnf sst tca "") + tpem=$(parse_cnf sst tcert "") + tkey=$(parse_cnf sst tkey "") + encrypt=$(parse_cnf sst encrypt 0) + sockopt=$(parse_cnf sst sockopt "") + progress=$(parse_cnf sst progress "") + rebuild=$(parse_cnf sst rebuild 0) + ttime=$(parse_cnf sst time 0) + cpat=$(parse_cnf sst cpat '.*galera\.cache$\|.*sst_in_progress$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$') + incremental=$(parse_cnf sst incremental 0) + ealgo=$(parse_cnf xtrabackup encrypt "") + ekey=$(parse_cnf xtrabackup encrypt-key "") + ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") + + # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html + if [[ -z $ealgo ]];then + ealgo=$(parse_cnf sst encrypt-algo "") + ekey=$(parse_cnf sst encrypt-key "") + ekeyfile=$(parse_cnf sst encrypt-key-file "") + fi + rlimit=$(parse_cnf sst rlimit "") + uextra=$(parse_cnf sst use_extra 0) +} + +get_stream() +{ + if [[ $sfmt == 'xbstream' ]];then + wsrep_log_info "Streaming with xbstream" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="xbstream -x" + else + strmcmd="xbstream -c \${INFO_FILE}" + fi + else + sfmt="tar" + wsrep_log_info "Streaming with tar" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="tar xfi - " + else + strmcmd="tar cf - \${INFO_FILE} " + fi + + fi +} + +get_proc() +{ + set +e + nproc=$(grep -c processor /proc/cpuinfo) + [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 + set -e +} + +sig_joiner_cleanup() +{ + wsrep_log_error "Removing $MAGIC_FILE file due to signal" + rm -f "$MAGIC_FILE" +} + +cleanup_joiner() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then + wsrep_log_info "Removing the sst_in_progress file" + wsrep_cleanup_progress_file + fi + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm $progress + fi + if [[ -n ${STATDIR:-} ]];then + [[ -d $STATDIR ]] && rm -rf $STATDIR + fi +} + +check_pid() +{ + local pid_file="$1" + [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 +} + +cleanup_donor() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + + if [[ -n $XTRABACKUP_PID ]];then + if check_pid $XTRABACKUP_PID + then + wsrep_log_error "xtrabackup process is still running. Killing... " + kill_xtrabackup + fi + + rm -f $XTRABACKUP_PID + fi + rm -f ${DATA}/${IST_FILE} + + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm $progress + fi +} + +kill_xtrabackup() +{ + local PID=$(cat $XTRABACKUP_PID) + [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : + rm -f "$XTRABACKUP_PID" +} + +setup_ports() +{ + if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then + SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }') + REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }') + lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }') + else + SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }') + fi +} + +# waits ~10 seconds for nc to open the port and then reports ready +# (regardless of timeout) +wait_for_listen() +{ + local PORT=$1 + local ADDR=$2 + local MODULE=$3 + for i in {1..50} + do + ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break + sleep 0.2 + done + if [[ $incremental -eq 1 ]];then + echo "ready ${ADDR}/${MODULE}/$lsn" + else + echo "ready ${ADDR}/${MODULE}" + fi +} + +check_extra() +{ + if [[ $uextra -eq 1 ]];then + if my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then + local eport=$(my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) + if [[ -n $eport ]];then + # Xtrabackup works only locally. + # Hence, setting host to 127.0.0.1 unconditionally. + wsrep_log_info "SST through extra_port $eport" + INNOEXTRA+=" --host=127.0.0.1 --port=$eport " + else + wsrep_log_error "Extra port $eport null, failing" + exit 1 + fi + else + wsrep_log_info "Thread pool not set, ignore the option use_extra" + fi + fi +} + +recv_joiner() +{ + local dir=$1 + local msg=$2 + + pushd ${dir} 1>/dev/null + set +e + timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + popd 1>/dev/null + + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + + if [ ! -r "${MAGIC_FILE}" ];then + # this message should cause joiner to abort + wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" + wsrep_log_info "Contents of datadir" + wsrep_log_info "$(ls -l ${dir}/*)" + exit 32 + fi +} + + +send_donor() +{ + local dir=$1 + local msg=$2 + + pushd ${dir} 1>/dev/null + set +e + timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + popd 1>/dev/null + + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + +} + +if [[ ! -x `which innobackupex` ]];then + wsrep_log_error "innobackupex not in path: $PATH" + exit 2 +fi + +rm -f "${MAGIC_FILE}" + +if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then + wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" + exit 22 +fi + +read_cnf +setup_ports +get_stream +get_transfer + +INNOEXTRA="" +INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" +INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \$INNOEXTRA --galera-info --stream=\$sfmt \${TMPDIR} 2>\${DATA}/innobackup.backup.log" + +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +then + trap cleanup_donor EXIT + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] + then + + TMPDIR="${TMPDIR:-/tmp}" + + if [ "${AUTH[0]}" != "(null)" ]; then + INNOEXTRA+=" --user=${AUTH[0]}" + fi + + if [ ${#AUTH[*]} -eq 2 ]; then + INNOEXTRA+=" --password=${AUTH[1]}" + elif [ "${AUTH[0]}" != "(null)" ]; then + # Empty password, used for testing, debugging etc. + INNOEXTRA+=" --password=" + fi + + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $ekey ]];then + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey " + else + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile " + fi + fi + + if [[ -n $lsn ]];then + INNOEXTRA+=" --incremental --incremental-lsn=$lsn " + fi + + check_extra + + wsrep_log_info "Streaming GTID file before SST" + + echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}" + + ttcmd="$tcmd" + + if [[ $encrypt -eq 1 ]];then + tcmd="$ecmd | $tcmd" + fi + + send_donor $DATA "${stagemsg}-gtid" + + tcmd="$ttcmd" + if [[ -n $progress ]];then + get_footprint + tcmd="$pcmd | $tcmd" + elif [[ -n $rlimit ]];then + adjust_progress + tcmd="$pcmd | $tcmd" + fi + + wsrep_log_info "Sleeping before data transfer for SST" + sleep 10 + + wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT:-4444}" + + set +e + timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + + if [ ${RC[0]} -ne 0 ]; then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ + "Check ${DATA}/innobackup.backup.log" + exit 22 + elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then + wsrep_log_error "$tcmd finished with error: ${RC[1]}" + exit 22 + fi + + # innobackupex implicitly writes PID to fixed location in ${TMPDIR} + XTRABACKUP_PID="${TMPDIR}/xtrabackup_pid" + + + else # BYPASS FOR IST + + wsrep_log_info "Bypassing the SST for IST" + echo "continue" # now server can resume updating data + echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}" + echo "1" > "${DATA}/${IST_FILE}" + get_keys + if [[ $encrypt -eq 1 ]];then + tcmd=" $ecmd | $tcmd" + fi + strmcmd+=" \${IST_FILE}" + + send_donor $DATA "${stagemsg}-IST" + + fi + + echo "done ${WSREP_SST_OPT_GTID}" + wsrep_log_info "Total time on donor: $totime seconds" + +elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] +then + [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE + + stagemsg="Joiner-Recv" + + if [[ ! -e ${DATA}/ibdata1 ]];then + incremental=0 + fi + + if [[ $incremental -eq 1 ]];then + wsrep_log_info "Incremental SST enabled" + #lsn=$(/pxc/bin/mysqld --defaults-file=$WSREP_SST_OPT_CONF --basedir=/pxc --wsrep-recover 2>&1 | grep -o 'log sequence number .*' | cut -d " " -f 4 | head -1) + lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ') + wsrep_log_info "Recovered LSN: $lsn" + fi + + sencrypted=1 + nthreads=1 + + MODULE="xtrabackup_sst" + + rm -f "${DATA}/${IST_FILE}" + + # May need xtrabackup_checkpoints later on + rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile + + ADDR=${WSREP_SST_OPT_ADDR} + if [ -z "${SST_PORT}" ] + then + SST_PORT=4444 + ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}" + fi + + wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} & + + trap sig_joiner_cleanup HUP PIPE INT TERM + trap cleanup_joiner EXIT + + if [[ -n $progress ]];then + adjust_progress + tcmd+=" | $pcmd" + fi + + if [[ $incremental -eq 1 ]];then + BDATA=$DATA + DATA=$(mktemp -d) + MAGIC_FILE="${DATA}/${INFO_FILE}" + fi + + get_keys + if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then + strmcmd=" $ecmd | $strmcmd" + fi + + STATDIR=$(mktemp -d) + MAGIC_FILE="${STATDIR}/${INFO_FILE}" + recv_joiner $STATDIR "${stagemsg}-gtid" 1 + + if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null + then + wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." + exit 32 + fi + + if [ ! -r "${STATDIR}/${IST_FILE}" ] + then + wsrep_log_info "Proceeding with SST" + + if [[ $incremental -ne 1 ]];then + wsrep_log_info "Cleaning the existing datadir" + find $DATA -mindepth 1 -regex $cpat -prune -o -exec rm -rfv {} 1>&2 \+ + else + wsrep_log_info "Removing existing ib_logfile files" + rm -f ${BDATA}/ib_logfile* + fi + + MAGIC_FILE="${DATA}/${INFO_FILE}" + recv_joiner $DATA "${stagemsg}-SST" 0 + + get_proc + + # Rebuild indexes for compact backups + if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then + wsrep_log_info "Index compaction detected" + rebuild=1 + fi + + if [[ $rebuild -eq 1 ]];then + nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) + wsrep_log_info "Rebuilding during prepare with $nthreads threads" + rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" + fi + + if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then + + wsrep_log_info "Compressed qpress files found" + + if [[ ! -x `which qpress` ]];then + wsrep_log_error "qpress not found in path: $PATH" + exit 22 + fi + + if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then + count=$(find ${DATA} -type f -name '*.qp' | wc -l) + count=$(( count*2 )) + if pv --help | grep -q FORMAT;then + pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" + else + pvopts="-f -s $count -l -N Decompression" + fi + pcmd="pv $pvopts" + adjust_progress + dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" + else + dcmd="xargs -n 2 qpress -T${nproc}d" + fi + + + # Decompress the qpress files + wsrep_log_info "Decompression with $nproc threads" + timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + extcode=$? + + if [[ $extcode -eq 0 ]];then + wsrep_log_info "Removing qpress files after decompression" + find ${DATA} -type f -name '*.qp' -delete + if [[ $? -ne 0 ]];then + wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" + fi + else + wsrep_log_error "Decompression failed. Exit code: $extcode" + exit 22 + fi + fi + + if [[ $incremental -eq 1 ]];then + # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues. + INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \ + --ibbackup=xtrabackup_55 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log" + fi + + wsrep_log_info "Preparing the backup at ${DATA}" + timeit "Xtrabackup prepare stage" "$INNOAPPLY" + + if [[ $incremental -eq 1 ]];then + wsrep_log_info "Cleaning up ${DATA} after incremental SST" + [[ -d ${DATA} ]] && rm -rf ${DATA} + DATA=$BDATA + fi + + if [ $? -ne 0 ]; + then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with errors. Check ${DATA}/innobackup.prepare.log" + exit 22 + fi + else + wsrep_log_info "${IST_FILE} received from donor: Running IST" + fi + + cat "${MAGIC_FILE}" # output UUID:seqno + wsrep_log_info "Total time on joiner: $totime seconds" +fi + +exit 0