PSTRESS-154 - Add RR support in pstress

Background:
===========

What is RR: Record and Replay tool

While using pstress, we come across bugs, crashes which are sporadic in nature. It may not be possible to easily reproduce these problems because of its flaky nature.
We have gdb (gnu debugger), which is an excellent tool if we can reproduce the problems, but if we run into crashes that occur in multi-threaded setup, we might not be able to repeat
the crash deterministicly

This is where RR comes handy. It has the ability to capture and then replay bugs. This tool has the ability to debug recorded sessions in a precise, deterministic fashion.

Author: mohitj1988

pstress/pstress-run-57.conf

@@ -185,3 +185,9 @@ GRP_RPL_CLUSTER_RUN=0
 # Default GR configuration file for multi-node pstress runs                    #
 ################################################################################
 GR_CLUSTER_CONFIG=${SCRIPT_PWD}/pstress-cluster-run.cfg
+
+################################################################################
+# To record and replay crashes, run pstress in RR mode                         #
+# To enable set RR_MODE=1                                                      #
+################################################################################
+RR_MODE=0

pstress/pstress-run-80.conf

@@ -189,3 +189,9 @@ GRP_RPL_CLUSTER_RUN=0
 # Default GR configuration file for multi-node pstress runs                    #
 ################################################################################
 GR_CLUSTER_CONFIG=${SCRIPT_PWD}/pstress-cluster-run.cfg
+
+################################################################################
+# To record and replay crashes, run pstress in RR mode                         #
+# To enable set RR_MODE=1                                                      #
+################################################################################
+RR_MODE=0

pstress/pstress-run-PXC57.conf

@@ -204,3 +204,9 @@ PXC_WSREP_PROVIDER_ADD_RANDOM_WSREP_PROVIDER_CONFIG_OPTIONS=0
 # Maximum number of PXC wsrep provider (Galera) configuration options to add   #
 ################################################################################
 PXC_WSREP_PROVIDER_MAX_NR_OF_RND_OPTS_TO_ADD=2
+
+################################################################################
+# To record and replay crashes, run pstress in RR mode                         #
+# To enable set RR_MODE=1                                                      #
+################################################################################
+RR_MODE=0

pstress/pstress-run-PXC80.conf

@@ -216,3 +216,9 @@ PXC_WSREP_PROVIDER_ADD_RANDOM_WSREP_PROVIDER_CONFIG_OPTIONS=0
 # Maximum number of PXC wsrep provider (Galera) configuration options to add   #
 ################################################################################
 PXC_WSREP_PROVIDER_MAX_NR_OF_RND_OPTS_TO_ADD=2
+
+################################################################################
+# To record and replay crashes, run pstress in RR mode                         #
+# To enable set RR_MODE=1                                                      #
+################################################################################
+RR_MODE=0

pstress/pstress-run-rocksdb.conf

@@ -152,3 +152,9 @@ MYINIT=
 # Extra options to pass to mysqld during server start                          #
 ################################################################################
 MYEXTRA=
+
+################################################################################
+# To record and replay crashes, run pstress in RR mode                         #
+# To enable set RR_MODE=1                                                      #
+################################################################################
+RR_MODE=0

pstress/pstress-run.sh

@@ -176,6 +176,24 @@ EOF
   fi
 }
 
+# Incase, user starts pstress in RR mode, check if RR is installed on the machine
+if [ $RR_MODE -eq 1 ]; then
+  echoit "Running pstress in RR mode. It is expected that pstress executions will be slower"
+  if [[ ! -e `which rr` ]];then
+    echo "rr package is not installed. Exiting"
+    echo "Install rr: https://github.com/rr-debugger/rr/wiki/Building-And-Installing"
+    exit 1
+  else
+    perf_event_var=$(cat /proc/sys/kernel/perf_event_paranoid)
+    if [ $perf_event_var -ne 1 ]; then
+      echo "rr needs /proc/sys/kernel/perf_event_paranoid <=1, but it is $perf_event_var"
+      echo "Change it to 1, consider running sudo sysctl -w kernel.perf_event_paranoid=1"
+      echo "For more information https://github.com/rr-debugger/rr/wiki/Building-And-Installing"
+      exit 1
+    fi
+  fi
+fi
+
 # Find mysqld binary
 if [ -r ${BASEDIR}/bin/mysqld ]; then
   BIN=${BASEDIR}/bin/mysqld
@@ -480,15 +498,27 @@ pxc_startup(){
   sed -i "2i wsrep_cluster_address=gcomm://${PXC_LADDRS[1]},${PXC_LADDRS[2]},${PXC_LADDRS[3]}" ${DATADIR}/n3.cnf
 
   get_error_socket_file 1
-  ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n1.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA --wsrep-new-cluster > ${ERR_FILE} 2>&1 &
+  if [ $RR_MODE -eq 1 ]; then
+    rr ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n1.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA --wsrep-new-cluster > ${ERR_FILE} 2>&1 &
+  elif [ $RR_MODE -eq 0 ]; then
+    ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n1.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA --wsrep-new-cluster > ${ERR_FILE} 2>&1 &
+  fi
   pxc_startup_status 1
 
   get_error_socket_file 2
-  ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n2.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  if [ $RR_MODE -eq 1 ]; then
+    rr ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n2.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  elif [ $RR_MODE -eq 0 ]; then
+    ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n2.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  fi
   pxc_startup_status 2
 
   get_error_socket_file 3
-  ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n3.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  if [ $RR_MODE -eq 1 ]; then
+    rr ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n3.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  elif [ $RR_MODE -eq 0 ]; then
+    ${BASEDIR}/bin/mysqld --defaults-file=${DATADIR}/n3.cnf $STARTUP_OPTION $MYEXTRA $PXC_MYEXTRA > ${ERR_FILE} 2>&1 &
+  fi
   pxc_startup_status 3
 
   if [ "$IS_STARTUP" == "startup" ]; then
@@ -905,6 +935,9 @@ pstress_test(){
         --log-output=none --log-error-verbosity=3 --log-error=${RUNDIR}/${TRIAL}/log/master.err"
     fi
 
+    if [ $RR_MODE -eq 1 ]; then
+      CMD="rr $CMD"
+    fi
     echo $CMD
     $CMD > ${RUNDIR}/${TRIAL}/log/master.err 2>&1 &
     MPID="$!"
@@ -1481,19 +1514,21 @@ elif [[ ${PXC} -eq 1 || ${GRP_RPL} -eq 1 ]]; then
     if ${BASEDIR}/bin/mysqladmin -uroot -S${WORKDIR}/node1.template/node1_socket.sock  ping > /dev/null 2>&1; then
       echoit "PXC node1.template started" ;
     else
-      echoit "Assert: PXC data template creation failed.."
+      echoit "Assert: PXC data template1 creation failed.."
       exit 1
     fi
+    sleep 2
     if ${BASEDIR}/bin/mysqladmin -uroot -S${WORKDIR}/node2.template/node2_socket.sock  ping > /dev/null 2>&1; then
       echoit "PXC node2.template started" ;
     else
-      echoit "Assert: PXC data template creation failed.."
+      echoit "Assert: PXC data template2 creation failed.."
       exit 1
     fi
+    sleep 2
     if ${BASEDIR}/bin/mysqladmin -uroot -S${WORKDIR}/node3.template/node3_socket.sock  ping > /dev/null 2>&1; then
       echoit "PXC node3.template started" ;
     else
-      echoit "Assert: PXC data template creation failed.."
+      echoit "Assert: PXC data template3 creation failed.."
       exit 1
     fi
     echoit "Created PXC data templates for pstress run.."