pquery-run.sh

#!/bin/bash
# Created by Roel Van de Paar, Percona LLC
# Updated by Ramesh Sivaraman, Percona LLC

# ========================================= Internal variables: !DO NOT CHANGE! ==================================================
RANDOM=`date +%s%N | cut -b14-19`; RANDOMD=$(echo $RANDOM$RANDOM$RANDOM | sed 's/..\(......\).*/\1/')
SCRIPT_AND_PATH=$(readlink -f $0); SCRIPT=$(echo ${SCRIPT_AND_PATH} | sed 's|.*/||'); SCRIPT_PWD=$(cd `dirname $0` && pwd)
WORKDIRACTIVE=0; SAVED=0; TRIAL=0; MYSQLD_START_TIMEOUT=60; STOREANYWAY=0

# ========================================= User configurable variablesi: generics ===============================================
PQUERY_BIN=${SCRIPT_PWD}/pquery/pquery-ps                           # pquery/pquery-ps: PS: Percona Server, pquery-ms: MS: MySQL, pquery-ws: WS/FB: WSS/fb-mysql, pquery-md: MD: MariaDB
INFILE=${SCRIPT_PWD}/pquery/main-ms-ps-md.sql                       # Default: main-ms-ps-md.sql (mtr_to_sql.sh of all major distro's + engine mix). Use main-ws.sql for mysql-fb
OPTIONS_INFILE=${SCRIPT_PWD}/pquery/mysqld_options_ps_57.txt        # Set to the right distribution + version to match mysqld options (as generated by generate_mysqld_options.sh)
TOKUDB_OPTIONS_INFILE=${SCRIPT_PWD}/pquery/mysqld_options_tokudb.txt # TokuDB options list
                                                                    # mysqld_options_ps_56.txt, mysqld_options_ms_56.txt, mysqld_options_ms_57.txt, mysqld_options_ps_55.txt etc.
BASEDIR=/sda/PS-19.01.16-mysql-5.7.10-1rc1-linux-x86_64-debug       # Basedir. Note that even for PXC runs, a PS basedir is needed (we need mysql client, mysqladmin,...)
WORKDIR=/sda/$RANDOMD                                               # Working directory. Here we keep the log files, option list, failed items. Please leave $RANDOMD!
RUNDIR=/dev/shm/$RANDOMD                                            # Run directory. Keeps a copy of the data dir template & stores mysqld runs (--datadir=...). Please leave $RANDOMD!
SYSBENCH_DATALOAD=0                                                 # Sysbench data load (normally off/0)

# ========================================= User configurable variablesi: generics ===============================================
TRIALS=100000                                                  # Number of individual trials to execute (one can always interrupt with ctrl+c also)
ADD_RANDOM_OPTIONS=1                                           # Add random mysqld --options to MYEXTRA using an input file. 1=On, 0=off
ADD_RANDOM_TOKUDB_OPTIONS=1                                    # Add random tokudb --options to MYEXTRA using an input file. 1=On, 0=off
MAX_NR_OF_RND_OPTS_TO_ADD=4                                    # Max nr of random options to add (minimum is always 1). Recommended: 4. Counts per ADD_RANDOM_... option above (x2)
SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY=1                       # Save only trials that generate a core file (good for initial few runs where there are lot of crashes/asserts)
SAVE_SQL=0                                                     # Saves per-trial SQL even if SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY=1. Main usecase: full server lockups/hangs
STORE_COPY_OF_INFILE=0                                         # Store a copy of the SQL input file (INFILE) in the work directory (this is not related to per-trial SQL)
PQUERY_RUN_TIMEOUT=15                                          # x sec max trial runtime within which pquery tries to process QUERIES_PER_THREAD x THREADS queries against 1 mysqld
QUERIES_PER_THREAD=100000                                      # Maximum number of queries executed per thread (THREADS) per trial (small = faster reduction, large = more crashes)
MYEXTRA=""                                                     # Extra options to pass to mysqld. Examples below
#MYEXTRA="--default-tmp-storage-engine=MyISAM --rocksdb --skip-innodb --default-storage-engine=RocksDB"                               # Use for RocksDB/fb-mysql testing
#MYEXTRA="--plugin-load-add=audit_log=audit_log.so --plugin-load-add=tokudb=ha_tokudb.so --init-file=${SCRIPT_PWD}/TokuDB.sql"        # Use for PS: TokuDB & audit-log testing
#MYEXTRA="--plugin-load-add=audit_log=audit_log.so --plugin-load-add=tokudb=ha_tokudb.so --init-file=${SCRIPT_PWD}/plugins.sql"       # Use for PS: enables all testable plugins
#MYEXTRA="--innodb_file_per_table=1 --innodb_flush_method=O_DIRECT --log-bin=binlog --binlog_format=MIXED"                            # Example of InnoDB & binlog options 
#MYEXTRA="--innodb_track_changed_pages"                                                                                               # Example of PS specific options
MYSAFE="--no-defaults --event-scheduler=ON --maximum-bulk_insert_buffer_size=1M --maximum-join_buffer_size=1M --maximum-max_heap_table_size=1M --maximum-max_join_size=1M --maximum-myisam_max_sort_file_size=1M --maximum-myisam_mmap_size=1M --maximum-myisam_sort_buffer_size=1M --maximum-optimizer_trace_max_mem_size=1M --maximum-preload_buffer_size=1M --maximum-query_alloc_block_size=1M --maximum-query_prealloc_size=1M --maximum-range_alloc_block_size=1M --maximum-read_buffer_size=1M --maximum-read_rnd_buffer_size=1M --maximum-sort_buffer_size=1M --maximum-tmp_table_size=1M --maximum-transaction_alloc_block_size=1M --maximum-transaction_prealloc_size=1M --log-output=none --sql_mode=ONLY_FULL_GROUP_BY"
                                                               # MYSAFE: See http://bugs.mysql.com/?id=73916 re: testing env setup. MYSAFE ensures cleaner/better QA runs
THREADS=1                                                      # Number of threads to use. Default: 1. Set to >1 to enable multi-threaded runs. Also see MULTI_THREADED_TESTC_LINES
MULTI_THREADED_TESTC_LINES=25000                               # Only takes effect if THREADS>1. pquery-run will take this amount of lines to form individual testcases for multi-threaded runs. IOW, if THREADS=1, pquery-run will just run one client, continously getting random queries out of INFILE, untill QUERIES_PER_THREAD queries is reached (if mysqld does not crash earlier). On the other hand, if THREADS>1, pquery-run will take a random chunk out of INFILE (to the amount of MULTI_THREADED_TESTC_LINES lines) _before_ starting each trial run. It will then run pquery, with THREADS threads, only using that smaller chunk extracted from INFILE. These individual SQL input files are saved in the trial's dirtory. You may want to keep this number small to ensure that true-multi-threaded testcase reduction using reducer.sh is still sufficiently fast (25K lines takes a few days to reduce), while keeping it large enough to ensure crashes/asserts to start with.

# ========================================= User configurable variables to enable query duration testing =========================
QUERY_DURATION_TESTING=0

# ========================================= User configurable variables to enable Valgrind testing only ==========================
VALGRIND_RUN=0                                                 # Set to 1 to make this a Valgrind run. Do not change VALGRIND_CMD unless you fully understand the change being made
VALGRIND_CMD="valgrind --suppressions=${BASEDIR}/mysql-test/valgrind.supp --num-callers=40 --show-reachable=yes --track-origins=yes"
VALGRIND_ERRORS_FOUND=0

# ========================================= User configurable variables to enable crash recovery testing only ====================
CRASH_RECOVERY_TESTING=0                                       # Set to 1 to make this a crash recovery test (terminates server after x seconds and then attempts crash recovery)
CRASH_RECOVERY_INFILE=${SCRIPT_PWD}/simple.sql                 # Currently using simple.sql for crash recovery testing. This can be changed/expanded upon
CRASH_RECOVERY_KILL_BEFORE_END_SEC=25                          # Kill the server x seconds before the end of the trial. Ensure trial duration is somewhat larger then this

# ========================================= User configurable variables to configure PS testing only =============================
SKIP_JEMALLOC_FOR_PS=0                                         # Skip LD_PRELOAD'ing of JEMALLOC. Only indicated for testing PS without TokuDB against default memory manager

# ========================================= User configurable variables to enable/for PXC testing only ===========================
PXC=0                                                          # Special use mode: Enable PXC testing
PXC_OPTIONS_INFILE=${SCRIPT_PWD}/pquery/pxc_mysqld_options.txt # PXC wsrep mysqld options
DOCKER_COMPOSE_LOC=${SCRIPT_PWD}/pxc-pquery/pquery-jenkins     # Only used for PXC runs which use Docker Compose
DOCKER_COMPOSE_YML=${SCRIPT_PWD}/pxc-pquery/pquery-jenkins/pqueryrun/docker-compose.yml
PXC_DOCKER_START_TIMEOUT=140                                   # Should not be necessary to change. Default: 140

# ========================================= Improvement ideas ====================================================================
# * SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY=0 (These likely include some of the 'SIGKILL' issues - no core but terminated)
# * SQL hashing s/t2/t1/, hex values "0x"
# * Full MTR grammar on one-liners
# * Interleave all statements with another that is likely to cause issues, for example "USE mysql"

# ========================================= Main code ============================================================================
# Security checks: ensure variables are correctly set to avoid rm -Rf issues (if not set correctly, it was likely due to altering internal variables at the top of this file)
if [ "${WORKDIR}" == "/sd[a-z][/]" ]; then echo "Assert! \$WORKDIR == '${WORKDIR}' - is it missing the \$RANDOMD suffix?"; exit 1; fi
if [ "${RUNDIR}" == "/dev/shm[/]" ]; then echo "Assert! \$RUNDIR == '${RUNDIR}' - is it missing the \$RANDOMD suffix?"; exit 1; fi
if [ "$(echo ${RANDOMD} | sed 's|[0-9]|/|g')" != "//////" ]; then echo "Assert! \$RANDOMD == '${RANDOMD}'. This looks incorrect - it should be 6 numbers exactly"; exit 1; fi
if [ "$(echo ${WORKDIR} | grep -oi "$RANDOMD" | head -n1)" != "${RANDOMD}" ]; then echo "Assert! \$WORKDIR == '${WORKDIR}' - is it missing the \$RANDOMD suffix?"; exit 1; fi
if [ "$(echo ${RUNDIR}  | grep -oi "$RANDOMD" | head -n1)" != "${RANDOMD}" ]; then echo "Assert! \$WORKDIR == '${WORKDIR}' - is it missing the \$RANDOMD suffix?"; exit 1; fi
if [ "$(echo ${PQUERY_BIN} | sed 's|\(^/pquery\)|\1|')" == "/pquery" ]; then echo "Assert! \$PQUERY_BIN == '${PQUERY_BIN}' - is it missing the \$SCRIPT_PWD prefix?"; exit 1; fi

# Output function
echoit(){
  echo "[$(date +'%T')] [$SAVED] $1"
  if [ ${WORKDIRACTIVE} -eq 1 ]; then echo "[$(date +'%T')] [$SAVED] $1" >> /${WORKDIR}/pquery-run.log; fi
}

# Find mysqld binary
if [ -r ${BASEDIR}/bin/mysqld ]; then
  BIN=${BASEDIR}/bin/mysqld
else
  # Check if this is a debug build by checking if debug string is present in dirname
  if [[ ${BASEDIR} = *debug* ]]; then
    if [ -r ${BASEDIR}/bin/mysqld-debug ]; then
      BIN=${BASEDIR}/bin/mysqld-debug
    else
      echoit "Something is wrong: there is no (script readable) mysqld binary at ${BASEDIR}/bin/mysqld[-debug] ?"
      exit 1
    fi
  else
    echoit "Something is wrong: there is no (script readable) mysqld binary at ${BASEDIR}/bin/mysqld ?"
    exit 1
  fi
fi

# JEMALLOC for PS/TokuDB
if [ ${SKIP_JEMALLOC_FOR_PS} -ne 1 ]; then
  if [ "$(${BIN} --version | grep -oi 'Percona' | sed 's|p|P|' | head -n1)" == "Percona" ] || [ `${BIN} --version | grep -oi '5.7.10-[0-9]' | cut -f2 -d'-' | head -n1` -ge 1 ]; then
    if [ -r `sudo find /usr/*lib*/ -name libjemalloc.so.1 | head -n1` ]; then
      export LD_PRELOAD=`sudo find /usr/*lib*/ -name libjemalloc.so.1 | head -n1`
    else 
      echoit "Assert! Binary (${BIN} reported itself as Percona Server, yet jemalloc was not found, please install it!";
      echoit "For Centos7 you can do this by:  sudo yum -y install epel-release; sudo yum -y install jemalloc;"
      echoit "For Ubuntu you can do this by: sudo apt-get install libjemalloc-dev;"
      exit 1; 
    fi
  fi
else
  if [ "$(${BIN} --version | grep -oi 'Percona' | sed 's|p|P|' | head -n1)" == "Percona" ] || [ `${BIN} --version | grep -oi '5.7.10-[0-9]' | cut -f2 -d'-' | head -n1` -ge 1 ]; then
    echoit "*** IMPORTANT WARNING ***: SKIP_JEMALLOC_FOR_PS was set to 1, and thus JEMALLOC will not be LD_PRELOAD'ed. However, the mysqld binary (${BIN}) reports itself as Percona Server. If you are going to test TokuDB, JEMALLOC should be LD_PRELOAD'ed. If not testing TokuDB, then this warning can be safely ignored."
  fi
fi

# Automatic variable adjustments
if [ "$1" == "pxc" -o "$2" == "pxc" -o "$1" == "PXC" -o "$2" == "PXC" ]; then PXC=1; fi  # Check if this is a a PXC run as indicated by first or second option to this script
if [ "$(whoami)" == "root" ]; then MYEXTRA="--user=root ${MYEXTRA}"; fi
if [ ${CRASH_RECOVERY_TESTING} -eq 1 ]; then INFILE=CRASH_RECOVERY_INFILE; fi
if [ ${PXC} -eq 1 ]; then
  if [ ${QUERIES_PER_THREAD} -lt 2147483647 ]; then  # Starting up a cluster takes more time, so don't rotate too quickly
    echoit "Note: As this is a PXC=1 run, and QUERIES_PER_THREAD was set to only ${QUERIES_PER_THREAD}, this script is setting the queries per thread to the required minimum of 2147483647 for this run."
    QUERIES_PER_THREAD=2147483647  # Max int
  fi
  if [ ${PQUERY_RUN_TIMEOUT} -lt 120 ]; then  # Starting up a cluster takes more time, so don't rotate too quickly
    echoit "Note: As this is a PXC=1 run, and PQUERY_RUN_TIMEOUT was set to only ${PQUERY_RUN_TIMEOUT}, this script is setting the timeout to the required minimum of 120 for this run."
    PQUERY_RUN_TIMEOUT=120
  fi
fi
if [ ${THREADS} -gt 1 ]; then  # We may want to drop this to 20 seconds required?
  if [ ${PQUERY_RUN_TIMEOUT} -lt 30 ]; then
    echoit "Note: As this is a multi-threaded run, and PQUERY_RUN_TIMEOUT was set to only ${PQUERY_RUN_TIMEOUT}, this script is setting the timeout to the required minimum of 30 for this run."
    PQUERY_RUN_TIMEOUT=30
  fi
fi
if [ ${CRASH_RECOVERY_TESTING} -eq 1 ]; then
  if [ ${THREADS} -lt 50 ]; then
    echoit "Note: As this is a CRASH_RECOVERY_TESTING=1 run, and THREADS was set to only ${THREADS}, this script is setting the number of threads to the required minimum of 50 for this run."
    THREADS=50
  fi
  if [ ${PQUERY_RUN_TIMEOUT} -lt 30 ]; then
    echoit "Note: As this is a CRASH_RECOVERY_TESTING=1 run, and PQUERY_RUN_TIMEOUT was set to only ${PQUERY_RUN_TIMEOUT}, this script is setting the timeout to the required minimum of 30 for this run."
    PQUERY_RUN_TIMEOUT=30
  fi
fi
if [ ${QUERY_DURATION_TESTING} -eq 1 ]; then
    echoit "Note: As this is a QUERY_DURATION_TESTING=1 run, and THREADS was set to ${THREADS}, this script is setting the number of threads to the required setting of 1 thread for this run. (While multi-threaded query duration testing may be added later, it's priority is low, and it's use case seems small.)"
    THREADS=1
fi
if [ ${VALGRIND_RUN} -eq 1 ]; then
  echoit "Note: As this is a VALGRIND_RUN=1 run, this script is increasing MYSQLD_START_TIMEOUT (${MYSQLD_START_TIMEOUT}) by 240 seconds because Valgrind is very slow in starting up mysqld."
  MYSQLD_START_TIMEOUT=$[ ${MYSQLD_START_TIMEOUT} + 240 ]
  if [ ${MYSQLD_START_TIMEOUT} -lt 300 ]; then
    echoit "Note: As this is a VALGRIND_RUN=1 run, and MYSQLD_START_TIMEOUT was set to only ${MYSQLD_START_TIMEOUT}), this script is setting the timeout to the required minimum of 300 for this run."
    MYSQLD_START_TIMEOUT=300
  fi
  echoit "Note: As this is a VALGRIND_RUN=1 run, this script is increasing PQUERY_RUN_TIMEOUT (${PQUERY_RUN_TIMEOUT}) by 180 seconds because Valgrind is very slow in processing SQL."
  PQUERY_RUN_TIMEOUT=$[ ${PQUERY_RUN_TIMEOUT} + 180 ] 
fi

# Trap ctrl-c 
trap ctrl-c SIGINT

ctrl-c(){
  echoit "CTRL+C Was pressed. Attempting to terminate running processes..."
  KILL_PIDS=`ps -ef | grep "$RANDOMD" | grep -v "grep" | awk '{print $2}' | tr '\n' ' '`
  if [ "${KILL_PIDS}" != "" ]; then
    echoit "Terminating the following PID's: ${KILL_PIDS}"
    kill -9 ${KILL_PIDS} >/dev/null 2>&1
  fi
  if [ -d ${RUNDIR}/${TRIAL}/ ]; then
    echoit "Done. Moving the trial $0 was currently working on to workdir as ${WORKDIR}/${TRIAL}/..."
    mv ${RUNDIR}/${TRIAL}/ ${WORKDIR}/
  fi
  echoit "Attempting to cleanup the pquery rundir ${RUNDIR}..."
  rm -Rf ${RUNDIR}
  if [ $SAVED -eq 0 -a ${SAVE_SQL} -eq 0 ]; then
    echoit "There were no coredumps saved, and SAVE_SQL=0, so the workdir can be safely deleted. Doing so..."
    rm -Rf ${WORKDIR}
  else
    echoit "The results of this run can be found in the workdir ${WORKDIR}..."
  fi
  echoit "Done. Terminating pquery-run.sh with exit code 2..."
  exit 2
}

savetrial(){
  if [ ${PXC} -eq 0 ]; then
    if [ -f ${RUNDIR}/${TRIAL}/data/*core* -o ${SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY} -eq 0 -o ${STOREANYWAY} -eq 1 -o ${VALGRIND_ERRORS_FOUND} -eq 1 ]; then
      SAVED=$[ $SAVED + 1 ]
      echoit "Copying rundir from ${RUNDIR}/${TRIAL} to ${WORKDIR}/${TRIAL}"
      mv ${RUNDIR}/${TRIAL}/ ${WORKDIR}/
    else
      echoit "Could not find core dump or Valgrind issue: Deleting rundir ${RUNDIR}/${TRIAL}"
      rm -Rf ${RUNDIR}/${TRIAL}
    fi
  else
    if [ $(ls -l ${RUNDIR}/${TRIAL}/*/*core.* 2>/dev/null | wc -l) -ge 1 ]; then
      SAVED=$[ $SAVED + 1 ]
      echoit "Copying rundir from ${RUNDIR}/${TRIAL} to ${WORKDIR}/${TRIAL}"
      mv ${RUNDIR}/${TRIAL}/ ${WORKDIR}/
    else
      echoit "Could not find core dump : Deleting rundir ${RUNDIR}/${TRIAL}"
      rm -Rf ${RUNDIR}/${TRIAL}
    fi
  fi
  STOREANYWAY=0
}

savesql(){
  echoit "Copying sql trace(s) from ${RUNDIR}/${TRIAL} to ${WORKDIR}/${TRIAL}"
  mkdir ${WORKDIR}/${TRIAL}
  cp ${RUNDIR}/${TRIAL}/*.sql ${WORKDIR}/${TRIAL}/
  rm -Rf ${RUNDIR}/${TRIAL}
  sync; sleep 0.2
  if [ -d ${RUNDIR}/${TRIAL} ]; then
    echoit "Something is wrong: tried to remove ${RUNDIR}/${TRIAL}, but it looks like removal failed. Check what is holding lock? (lsof tool may help)."
    echoit "As this is not necessarily a fatal error (there is likely enough space on ${RUNDIR} to continue working), pquery-run.sh will NOT terminate."
    echoit "However, this looks like a shortcoming in pquery-run.sh (likely in the mysqld termination code) which needs debugging and fixing. Please do."
  fi
}

pquery_test(){
  TRIAL=$[ ${TRIAL} + 1 ]
  echoit "====== TRIAL #${TRIAL} ======"
  if [ ${PXC} -eq 0 ]; then
    echoit "Ensuring there are no relevant servers running..."
    KILLPID=$(ps -ef | grep "${RUNDIR}" | grep -v grep | awk '{print $2}' | tr '\n' ' ')
    (sleep 0.2; kill -9 $KILLPID >/dev/null 2>&1; wait $KILLPID >/dev/null 2>&1) &
    wait $KILLDPID >/dev/null 2>&1  # The sleep 0.2 + subsequent wait (cought before the kill) avoids the annoying 'Killed' message 
                                    # from being displayed in the output. Thank you to user 'Foonly' @ forums.whirlpool.net.au
  else
    echoit "Ensuring there are no relevant Docker containers present..."
    if [ ! -r ${DOCKER_COMPOSE_LOC}/cleanup.sh ]; then
      echoit "Assert: ${DOCKER_COMPOSE_LOC}/cleanup.sh was not found!"
      exit 1
    else
      ${DOCKER_COMPOSE_LOC}/cleanup.sh
    fi
  fi
  echoit "Clearing rundir..."
  rm -Rf ${RUNDIR}/*
  echoit "Generating new trial workdir ${RUNDIR}/${TRIAL}..."
  ISSTARTED=0
  if [ ${PXC} -eq 0 ]; then
    mkdir -p ${RUNDIR}/${TRIAL}/data/test ${RUNDIR}/${TRIAL}/data/mysql ${RUNDIR}/${TRIAL}/tmp ${RUNDIR}/${TRIAL}/log
    echo 'SELECT 1;' > ${RUNDIR}/${TRIAL}/pquery_thread-0.sql  # Add fake pquery_thread-0.sql file enabling pquery-prep-red.sh/reducer.sh to be used with/for mysqld startup issues
    echoit "Copying datadir from template..."
    if [ `ls -l ${WORKDIR}/data.template/* | wc -l` -eq 0 ]; then
      echoit "Assert: ${WORKDIR}/data.template/ is empty? Check ${WORKDIR}/log/mysql_install_db.txt to see if the original template creation worked ok. Terminating."
      echoit "Note that this is can be caused by not having perl-Data-Dumper installed (sudo yum install perl-Data-Dumper), which is required for mysql_install_db."
      exit 1
    fi
    cp -R ${WORKDIR}/data.template/* ${RUNDIR}/${TRIAL}/data
    MYEXTRA_SAVE_IT=${MYEXTRA}
    if [ ${ADD_RANDOM_OPTIONS} -eq 1 ]; then  # Add random mysqld --options to MYEXTRA
      OPTIONS_TO_ADD=
      NR_OF_OPTIONS_TO_ADD=$(( RANDOM % MAX_NR_OF_RND_OPTS_TO_ADD + 1 ))
      for X in $(seq 1 ${NR_OF_OPTIONS_TO_ADD}); do
        OPTION_TO_ADD="$(shuf --random-source=/dev/urandom ${OPTIONS_INFILE} | head -n1)"
        if [ "$(echo ${OPTION_TO_ADD} | sed 's| ||g;s|.*query.alloc.block.size=1125899906842624.*||' )" != "" ]; then  # http://bugs.mysql.com/bug.php?id=78238
          OPTIONS_TO_ADD="${OPTIONS_TO_ADD} ${OPTION_TO_ADD}"
        fi
      done
      echoit "ADD_RANDOM_OPTIONS=1: adding option(s) ${OPTIONS_TO_ADD} to this run's MYEXTRA..."
      MYEXTRA="${MYEXTRA} ${OPTIONS_TO_ADD}"
    fi
    if [ ${ADD_RANDOM_TOKUDB_OPTIONS} -eq 1 ]; then  # Add random tokudb --options to MYEXTRA
      OPTIONS_TO_ADD=
      NR_OF_OPTIONS_TO_ADD=$(( RANDOM % MAX_NR_OF_RND_OPTS_TO_ADD + 1 ))
      for X in $(seq 1 ${NR_OF_OPTIONS_TO_ADD}); do
        OPTION_TO_ADD="$(shuf --random-source=/dev/urandom ${TOKUDB_OPTIONS_INFILE} | head -n1)"
        OPTIONS_TO_ADD="${OPTIONS_TO_ADD} ${OPTION_TO_ADD}"
      done
      echoit "ADD_RANDOM_TOKUDB_OPTIONS=1: adding option(s) ${OPTIONS_TO_ADD} to this run's MYEXTRA..."
      MYEXTRA="${MYEXTRA} ${OPTIONS_TO_ADD}"
    fi
    echo "${MYEXTRA}" | if grep -qi "innodb[_-]log[_-]checksum[_-]algorithm"; then
      # Ensure that mysqld server startup will not fail due to a mismatched checksum algo between the original MID and the changed MYEXTRA options
      rm ${RUNDIR}/${TRIAL}/data/ib_log*
    fi
    PORT=$[50000 + ( $RANDOM % ( 9999 ) ) ]
    echoit "Starting mysqld. Error log is stored at ${RUNDIR}/${TRIAL}/log/master.err"
    if [ ${VALGRIND_RUN} -eq 0 ]; then
      CMD="${BIN} ${MYSAFE} ${MYEXTRA} --basedir=${BASEDIR} --datadir=${RUNDIR}/${TRIAL}/data --tmpdir=${RUNDIR}/${TRIAL}/tmp \
	--core-file --port=$PORT --pid_file=${RUNDIR}/${TRIAL}/pid.pid --socket=${RUNDIR}/${TRIAL}/socket.sock \
	--log-output=none --log-error=${RUNDIR}/${TRIAL}/log/master.err"
    else
      CMD="${VALGRIND_CMD} ${BIN} ${MYSAFE} ${MYEXTRA} --basedir=${BASEDIR} --datadir=${RUNDIR}/${TRIAL}/data --tmpdir=${RUNDIR}/${TRIAL}/tmp \
	--core-file --port=$PORT --pid_file=${RUNDIR}/${TRIAL}/pid.pid --socket=${RUNDIR}/${TRIAL}/socket.sock \
	--log-output=none --log-error=${RUNDIR}/${TRIAL}/log/master.err"
    fi
    $CMD > ${RUNDIR}/${TRIAL}/log/master.err 2>&1 &
    MPID="$!"
    echo "## Good for reproducing mysqld (5.7+) startup issues only (full issues need a data dir, so use mysql_install_db or mysqld --init for those)" > ${RUNDIR}/${TRIAL}/start
    echo "echo '=== Setting up directories...'" >> ${RUNDIR}/${TRIAL}/start
    echo "rm -Rf ${RUNDIR}/${TRIAL}" >> ${RUNDIR}/${TRIAL}/start
    echo "mkdir -p ${RUNDIR}/${TRIAL}/data ${RUNDIR}/${TRIAL}/tmp" >> ${RUNDIR}/${TRIAL}/start
    echo "echo '=== Data dir init...'" >> ${RUNDIR}/${TRIAL}/start
    echo "${BIN} --no-defaults --initialize-insecure --basedir=${BASEDIR} --datadir=${RUNDIR}/${TRIAL}/data --tmpdir=${RUNDIR}/${TRIAL}/tmp --core-file --port=$PORT --pid_file=${RUNDIR}/${TRIAL}/pid.pid --socket=${RUNDIR}/${TRIAL}/socket.sock --log-output=none --log-error=${RUNDIR}/${TRIAL}/log/master.err" >> ${RUNDIR}/${TRIAL}/start
    echo "echo '=== Starting mysqld...'" >> ${RUNDIR}/${TRIAL}/start
    echo "${CMD} > ${RUNDIR}/${TRIAL}/log/master.err 2>&1" >> ${RUNDIR}/${TRIAL}/start
    echo "# Same startup command, but without MYEXTRA included:" >> ${RUNDIR}/${TRIAL}/start 
    echo "#$(echo ${CMD} | sed 's|${MYEXTRA}||')" >> ${RUNDIR}/${TRIAL}/start
    chmod +x ${RUNDIR}/${TRIAL}/start
    echo "BASEDIR=$BASEDIR" > ${RUNDIR}/${TRIAL}/start_recovery
    echo "if [ -r /usr/lib64/libjemalloc.so.1 ]; then" >> ${RUNDIR}/${TRIAL}/start_recovery
    echo "  export LD_PRELOAD=/usr/lib64/libjemalloc.so.1;" >> ${RUNDIR}/${TRIAL}/start_recovery
    echo "else" >> ${RUNDIR}/${TRIAL}/start_recovery
    echo "  echo \"Assert! jemalloc not found at /usr/lib64/libjemalloc.so.1, please install it!\";" >> ${RUNDIR}/${TRIAL}/start_recovery
    echo "  echoit \"For Centos7 you can do this by: sudo yum -y install epel-release; sudo yum -y install jemalloc;\"" >> ${RUNDIR}/${TRIAL}/start_recovery
    echo "  exit 1;" >> ${RUNDIR}/${TRIAL}/start_recovery
    echo "fi" >> ${RUNDIR}/${TRIAL}/start_recovery
    
    echo "${CMD//$RUNDIR/$WORKDIR} --init-file=${WORKDIR}/recovery-user.sql > ${WORKDIR}/${TRIAL}/log/master.err 2>&1 &" >> ${RUNDIR}/${TRIAL}/start_recovery ; chmod +x ${RUNDIR}/${TRIAL}/start_recovery
    # New MYEXTRA/MYSAFE variables pass & VALGRIND run check method as of 2015-07-28 (MYSAFE & MYEXTRA stored in a text file inside the trial dir, VALGRIND file created if used)
    echo "${MYSAFE} ${MYEXTRA}" > ${RUNDIR}/${TRIAL}/MYEXTRA
    if [ ${VALGRIND_RUN} -eq 1 ]; then
      touch  ${RUNDIR}/${TRIAL}/VALGRIND
    fi
    # Restore orignal MYEXTRA for the next trial (MYEXTRA is no longer needed anywhere else. If this changes in the future, relocate this to below the changed code)
    MYEXTRA=${MYEXTRA_SAVE_IT}  
    # Give up to x (start timeout) seconds for mysqld to start, but check intelligently for known startup issues like "Error while setting value" for options
    if [ ${VALGRIND_RUN} -eq 0 ]; then
      echoit "Waiting for mysqld (pid: ${MPID}) to fully start..."
    else
      echoit "Waiting for mysqld (pid: ${MPID}) to fully start (note this is slow for Valgrind runs, and can easily take 35-90 seconds even on an high end server)..."
    fi
    BADVALUE=0
    for X in $(seq 0 ${MYSQLD_START_TIMEOUT}); do
      sleep 1
      if ${BASEDIR}/bin/mysqladmin -uroot -S${RUNDIR}/${TRIAL}/socket.sock ping > /dev/null 2>&1; then
        break
      fi
      if [ "${MPID}" == "" ]; then
        echoit "Assert! ${MPID} empty. Terminating!"
        exit 1
      fi
      if grep -qi "ERROR. Aborting" ${RUNDIR}/${TRIAL}/log/master.err; then
        if grep -qi "TCP.IP port. Address already in use" ${RUNDIR}/${TRIAL}/log/master.err; then
          echoit "Assert! The text '[ERROR] Aborting' was found in the error log due to a IP port conflict (the port was already in use)"
          savetrial
        else
          if [ ${ADD_RANDOM_OPTIONS} -eq 0 ]; then  # Halt for ADD_RANDOM_OPTIONS=0 runs, they should not produce errors like these, as MYEXTRA should be high-quality/non-faulty
            echoit "Assert! '[ERROR] Aborting' was found in the error log. This is likely an issue with one of the \$MEXTRA (or \$MYSAFE) startup parameters. Saving trial for further analysis, and dumping error log here for quick analysis. Please check the output against the \$MYEXTRA (or \$MYSAFE if it was modified) settings. You may also want to try setting \$MYEXTRA=\"\"..."
            grep "ERROR" ${RUNDIR}/${TRIAL}/log/master.err | tee -a /${WORKDIR}/pquery-run.log
            STOREANYWAY=1
            savetrial
            echoit "Remember to cleanup/delete the rundir:  rm -Rf ${RUNDIR}"
            exit 1
          else  # Do not halt for ADD_RANDOM_OPTIONS=1 runs, they are likely to produce errors like these as MYEXTRA was randomly changed
            echoit "'[ERROR] Aborting' was found in the error log. This is likely an issue with one of the MYEXTRA startup parameters. As ADD_RANDOM_OPTIONS=1, this is likely to be encountered. Not saving trial. If you see this error for every trial however, set \$ADD_RANDOM_OPTIONS=0 & try running pquery-run.sh again. If it still fails, your base \$MYEXTRA setting is faulty."
            grep "ERROR" ${RUNDIR}/${TRIAL}/log/master.err | tee -a /${WORKDIR}/pquery-run.log
            break
          fi
        fi
      fi
      if [ $(ls -l ${RUNDIR}/${TRIAL}/*/*core.* 2>/dev/null | wc -l) -ge 1 ]; then break; fi  # Break the wait-for-server-started loop if a core file is found. Handling of core is done below.
    done
    # Check if mysqld is alive and if so, set ISSTARTED=1 so pquery will run
    if ${BASEDIR}/bin/mysqladmin -uroot -S${RUNDIR}/${TRIAL}/socket.sock ping > /dev/null 2>&1; then
      ISSTARTED=1
      echoit "Server started ok. Client:  `echo ${BIN} | sed 's|/mysqld|/mysql|'` -uroot -S${RUNDIR}/${TRIAL}/socket.sock"
    fi
  else
    mkdir -p ${RUNDIR}/${TRIAL}/
    chmod 777 -R ${RUNDIR} 
    echoit "Copying docker-compose file (${DOCKER_COMPOSE_YML}) to trial workdir ${RUNDIR}/${TRIAL}"
    if [ ! -r ${DOCKER_COMPOSE_YML} ]; then
      echoit "Assert: ${DOCKER_COMPOSE_YML} not found? Terminating."
      exit 1
    else
      cp ${DOCKER_COMPOSE_YML} ${RUNDIR}/${TRIAL} 
    fi

    # Adding random PXC wsrep related mysqld options to MYEXTRA 
    MYEXTRA_SAVE_IT=${MYEXTRA}
    if [ ${ADD_RANDOM_OPTIONS} -eq 1 ]; then  # Add random mysqld --options to MYEXTRA
      OPTIONS_TO_ADD=
      NR_OF_OPTIONS_TO_ADD=$(( RANDOM % MAX_NR_OF_RND_OPTS_TO_ADD + 1 ))
      for X in $(seq 1 ${NR_OF_OPTIONS_TO_ADD}); do
        OPTION_TO_ADD="$(shuf --random-source=/dev/urandom ${PXC_OPTIONS_INFILE} | head -n1)"
        OPTIONS_TO_ADD="${OPTIONS_TO_ADD} ${OPTION_TO_ADD}"
      done
      echoit "ADD_RANDOM_OPTIONS=1: adding option(s) ${OPTIONS_TO_ADD} to this run's MYEXTRA..."
      MYEXTRA=" ${MYEXTRA} ${OPTIONS_TO_ADD} --log-error=error.log"
      sed -i "s|--log-error=error.log|${MYEXTRA}|" ${RUNDIR}/${TRIAL}/docker-compose.yml
    fi
    echo "${MYSAFE} ${MYEXTRA}" > ${RUNDIR}/${TRIAL}/MYEXTRA
    # Restore orignal MYEXTRA for the next trial (MYEXTRA is no longer needed anywhere else. If this changes in the future, relocate this to below the changed code)
    MYEXTRA=${MYEXTRA_SAVE_IT}
    echoit "Changing ${RUNDIR}/${TRIAL}/docker-compose.yml to use this trial's path (${RUNDIR}/${TRIAL})..."
    sed -i "s|/dev/shm/pxc-pquery|${RUNDIR}/${TRIAL}|" ${RUNDIR}/${TRIAL}/docker-compose.yml
    echoit "Starting 3 node PXC Cluster..."
    CURPATH=$PWD
    cd ${RUNDIR}/${TRIAL}
    sudo docker-compose up &
    cd ${CURPATH}; CURPATH=
    echoit "Waiting for the 3 node PXC Cluster to fully start..."
    for X in $(seq 0 ${PXC_DOCKER_START_TIMEOUT}); do
      sleep 1
      CLUSTER_UP=0;
      if ${BASEDIR}/bin/mysqladmin -uroot -h127.0.0.1 -P12000 ping > /dev/null 2>&1; then
        if [ `${BASEDIR}/bin/mysql -uroot -h127.0.0.1 -P10000 -e"show global status like 'wsrep_cluster_size'" | sed 's/[| \t]\+/\t/g' | grep "wsrep_cluster" | awk '{print $2}'` -eq 3 ]; then CLUSTER_UP=$[ ${CLUSTER_UP} + 1]; fi
        if [ `${BASEDIR}/bin/mysql -uroot -h127.0.0.1 -P11000 -e"show global status like 'wsrep_cluster_size'" | sed 's/[| \t]\+/\t/g' | grep "wsrep_cluster" | awk '{print $2}'` -eq 3 ]; then CLUSTER_UP=$[ ${CLUSTER_UP} + 1]; fi
        if [ `${BASEDIR}/bin/mysql -uroot -h127.0.0.1 -P12000 -e"show global status like 'wsrep_cluster_size'" | sed 's/[| \t]\+/\t/g' | grep "wsrep_cluster" | awk '{print $2}'` -eq 3 ]; then CLUSTER_UP=$[ ${CLUSTER_UP} + 1]; fi
        if [ "`${BASEDIR}/bin/mysql -uroot -h127.0.0.1 -P10000 -e"show global status like 'wsrep_local_state_comment'" | sed 's/[| \t]\+/\t/g' | grep "wsrep_local" | awk '{print $2}'`" == "Synced" ]; then CLUSTER_UP=$[ ${CLUSTER_UP} + 1]; fi
        if [ "`${BASEDIR}/bin/mysql -uroot -h127.0.0.1 -P11000 -e"show global status like 'wsrep_local_state_comment'" | sed 's/[| \t]\+/\t/g' | grep "wsrep_local" | awk '{print $2}'`" == "Synced" ]; then CLUSTER_UP=$[ ${CLUSTER_UP} + 1]; fi
        if [ "`${BASEDIR}/bin/mysql -uroot -h127.0.0.1 -P12000 -e"show global status like 'wsrep_local_state_comment'" | sed 's/[| \t]\+/\t/g' | grep "wsrep_local" | awk '{print $2}'`" == "Synced" ]; then CLUSTER_UP=$[ ${CLUSTER_UP} + 1]; fi
      fi
      # If count reached 6 (there are 6 checks), then the Cluster is up & running and consistent in it's Cluster topology views (as seen by each node)
      if [ ${CLUSTER_UP} -eq 6 ]; then
        ISSTARTED=1
        echoit "3 Node PXC Cluster started ok. Clients:"
        echoit "Node #1: `echo ${BIN} | sed 's|/mysqld|/mysql|'` -uroot -h127.0.0.1 -P10000"
        echoit "Node #2: `echo ${BIN} | sed 's|/mysqld|/mysql|'` -uroot -h127.0.0.1 -P11000"
        echoit "Node #3: `echo ${BIN} | sed 's|/mysqld|/mysql|'` -uroot -h127.0.0.1 -P12000"
        break
      fi
    done
  fi
 
  if [ ${ISSTARTED} -eq 1 ]; then
    rm -f ${RUNDIR}/${TRIAL}/pquery_thread-0.sql  # Remove the earlier created fake (SELECT 1; only) pquery_thread-0.sql file present for startup issues (server is started OK now)
    echoit "Starting pquery (log stored in ${RUNDIR}/${TRIAL}/pquery.log)..."
    if [ ${THREADS} -eq 1 ]; then  # Single-threaded run (1 client only)
      if [ ${PXC} -eq 0 ]; then
        if [ ${QUERY_DURATION_TESTING} -eq 0 ]; then
          ${PQUERY_BIN} --infile=${INFILE} --database=test --threads=${THREADS} --queries-per-thread=${QUERIES_PER_THREAD} --logdir=${RUNDIR}/${TRIAL} --log-all-queries --log-failed-queries --user=root --socket=${RUNDIR}/${TRIAL}/socket.sock >${RUNDIR}/${TRIAL}/pquery.log 2>&1 &
        else
          ${PQUERY_BIN} --infile=${INFILE} --database=test --threads=${THREADS} --queries-per-thread=${QUERIES_PER_THREAD} --logdir=${RUNDIR}/${TRIAL} --log-all-queries --log-failed-queries --log-query-duration --user=root --socket=${RUNDIR}/${TRIAL}/socket.sock >${RUNDIR}/${TRIAL}/pquery.log 2>&1 &
        fi
      else
        ${PQUERY_BIN} --infile=${INFILE} --database=test --threads=${THREADS} --queries-per-thread=${QUERIES_PER_THREAD} --logdir=${RUNDIR}/${TRIAL} --log-all-queries --log-failed-queries --user=root --addr=127.0.0.1 --port=10000 >${RUNDIR}/${TRIAL}/pquery.log 2>&1 &
      fi
    else  
      if [ ${CRASH_RECOVERY_TESTING} -eq 1 ]; then
        SQL_FILE="--infile=${INFILE}"
      else
        # Multi-threaded run using a chunk from INFILE (${THREADS} clients)
        echoit "Taking ${MULTI_THREADED_TESTC_LINES} lines randomly from ${INFILE} as testcase for this trial..."
        shuf --random-source=/dev/urandom ${INFILE} | head -n${MULTI_THREADED_TESTC_LINES} > ${RUNDIR}/${TRIAL}/${TRIAL}.sql
        SQL_FILE="--infile=${RUNDIR}/${TRIAL}/${TRIAL}.sql"
      fi
      # Debug echo "-------"; cat ${RUNDIR}/${TRIAL}/${TRIAL}.sql; echo "-------"
      if [ ${PXC} -eq 0 ]; then
        ${PQUERY_BIN} ${SQL_FILE} --database=test --threads=${THREADS} --queries-per-thread=${QUERIES_PER_THREAD} --logdir=${RUNDIR}/${TRIAL} --log-all-queries --log-failed-queries --user=root --socket=${RUNDIR}/${TRIAL}/socket.sock >${RUNDIR}/${TRIAL}/pquery.log 2>&1 &
      else
        ${PQUERY_BIN} ${SQL_FILE} --database=test --threads=${THREADS} --queries-per-thread=${QUERIES_PER_THREAD} --logdir=${RUNDIR}/${TRIAL} --log-all-queries --log-failed-queries --user=root --addr=127.0.0.1 --port=10000 >${RUNDIR}/${TRIAL}/pquery.log 2>&1 &
      fi
    fi 
    PQPID="$!"
    echoit "pquery running (Max duration: ${PQUERY_RUN_TIMEOUT}s)..."
    for X in $(seq 0 ${PQUERY_RUN_TIMEOUT}); do
      if grep -qi "error while loading shared libraries" ${RUNDIR}/${TRIAL}/pquery.log; then
        echoit "Assert: There was an error loading the shared/dynamic mysql client library linked to from within pquery. Ref. ${RUNDIR}/${TRIAL}/pquery.log to see the error. The solution is to ensure that LD_LIBRARY_PATH is set correctly (for example: execute '$ export LD_LIBRARY_PATH=<your_mysql_base_directory>/lib' in your shell. This will happen only if you use pquery without statically linked client libraries, and this in turn would happen only if you compiled pquery yourself instead of using the pre-built binaries available in https://github.com/Percona-QA/percona-qa (ref subdirectory/files ./pquery/pquery*) - which are normally used by this script (hence this situation is odd to start with). The pquery binaries in percona-qa all include a statically linked mysql client library matching the mysql flavor (PS,MS,MD,WS) it was built for. Another reason for this error may be that (having used pquery without statically linked client binaries as mentioned earlier) the client libraries are not available at the location set in LD_LIBRARY_PATH (which is currently set to '${LD_LIBRARY_PATH}'."
        exit 1
      fi
      Y=$X
      sleep 1
      if [ "`ps -ef | grep ${PQPID} | grep -v grep`" == "" ]; then  # pquery ended
        break
      fi
      if [ ${CRASH_RECOVERY_TESTING} -eq 1 ]; then
        if [ $X -eq $CRASH_RECOVERY_KILL_BEFORE_END_SEC ]; then
           kill -9 ${MPID} >/dev/null 2>&1;
           sleep 2
           echoit "killed for crash testing"
           break
        fi
      fi
    done
    if [ $Y -eq ${PQUERY_RUN_TIMEOUT} ]; then 
      echoit "${PQUERY_RUN_TIMEOUT}s timeout reached. Terminating this trial..."
    fi
  else
    if [ ${PXC} -eq 0 ]; then
      echoit "Server (PID: ${MPID} | Socket: ${RUNDIR}/${TRIAL}/socket.sock) failed to start after ${MYSQLD_START_TIMEOUT} seconds. Will issue extra kill -9 to ensure it's gone..."
      (sleep 0.2; kill -9 ${MPID} >/dev/null 2>&1; wait ${MPID} >/dev/null 2>&1) &
      wait ${MPID} >/dev/null 2>&1
      sleep 2; sync
    else
      echoit "3 Node PXC Cluster failed to start after ${PXC_DOCKER_START_TIMEOUT} seconds. Will issue an extra cleanup to ensure nothing remains..."
      ${DOCKER_COMPOSE_LOC}/cleanup.sh
      sleep 2; sync
    fi
  fi
  if [ ${VALGRIND_RUN} -eq 1 ]; then
    echoit "Cleaning up & saving results as needed. Note that this may take up to 10 minutes because this is a Valgrind run. You may also see a mysqladmin killed message..."
  else
    echoit "Cleaning up & saving results as needed..."
  fi
  TRIAL_SAVED=0;
  sleep 2  # Delay to ensure core was written completely (if any)
  if [ ${PXC} -eq 0 ]; then
    if [ ${VALGRIND_RUN} -eq 1 ]; then # For Valgrind, we want the full Valgrind output in the error log, hence we need a proper/clean (and slow...) shutdown
      # Note that even if mysqladmin is killed with the 'timeout --signal=9', it will not affect the actual state of mysqld, all that was terminated was mysqladmin. 
      # Thus, mysqld would (presumably) have received a shutdown signal (even if the timeout was 2 seconds it likely would have)
      timeout --signal=9 20s ${BASEDIR}/bin/mysqladmin -uroot -S${RUNDIR}/${TRIAL}/socket.sock shutdown > /dev/null 2>&1  # Proper/clean shutdown attempt (up to 20 sec wait), necessary to get full Valgrind output in error log
      VALGRIND_SUMMARY_FOUND=0
      for X in $(seq 0 600); do  # Wait for full Valgrind output in error log
        sleep 1
        if [ ! -r ${RUNDIR}/${TRIAL}/log/master.err ]; then
          echoit "Assert: ${RUNDIR}/${TRIAL}/log/master.err not found during a Valgrind run. Please check. Trying to continue, but something is wrong already..."
          break
        elif egrep -qi "==[0-9]+== ERROR SUMMARY: [0-9]+ error" ${RUNDIR}/${TRIAL}/log/master.err; then  # Summary found, Valgrind is done
          VALGRIND_SUMMARY_FOUND=1
          sleep 2
          break
        fi
      done
      if [ ${VALGRIND_SUMMARY_FOUND} -eq 0 ]; then
        kill -9 ${PQPID} >/dev/null 2>&1;
        kill -9 ${MPID} >/dev/null 2>&1;
        sleep 2  # <^ Make sure mysqld is gone
        echoit "Odd mysqld hang detected (mysqld did not terminate even after 600 seconds), saving this trial... "
        if [ ${TRIAL_SAVED} -eq 0 ]; then
          savetrial
          TRIAL_SAVED=1
        fi
      fi
    else
      timeout --signal=9 20s ${BASEDIR}/bin/mysqladmin -uroot -S${RUNDIR}/${TRIAL}/socket.sock shutdown > /dev/null 2>&1  # Proper/clean shutdown attempt (up to 20 sec wait), may catch shutdown bugs
      sleep 2
    fi
    (sleep 0.2; kill -9 ${MPID} >/dev/null 2>&1; wait ${MPID} >/dev/null 2>&1) &  # Terminate mysqld
    wait ${MPID} >/dev/null 2>&1
    (sleep 0.2; kill -9 ${PQPID} >/dev/null 2>&1; wait ${PQPID} >/dev/null 2>&1) &  # Terminate pquery (if it went past ${PQUERY_RUN_TIMEOUT} time)
    wait ${PQPID} >/dev/null 2>&1
    kill -9 ${PQPID} >/dev/null 2>&1;
    kill -9 ${MPID} >/dev/null 2>&1;
    sleep 2  # <^ Make sure mysqld is gone
  else
    ${SCRIPT_PWD}/pxc-pquery/cleanup.sh
    echoit "Changing owner of trial workdir (${RUNDIR}/${TRIAL}) to `whoami`:`whoami`..."
    sudo chown -R `whoami`:`whoami` ${RUNDIR}/${TRIAL}
  fi
  if [ ${ISSTARTED} -eq 1 ]; then  # Do not try and print pquery log for a failed mysqld start
    echoit "$(cat ${RUNDIR}/${TRIAL}/pquery.log | grep -i 'SUMMARY' | sed 's|^.*:|pquery summary:|')"
  fi
  if [ ${VALGRIND_RUN} -eq 1 ]; then
    VALGRIND_ERRORS_FOUND=0; VALGRIND_CHECK_1=
    # What follows next are 3 different ways of checking if Valgrind issues were seen, mostly to ensure that no Valgrind issues go unseen, especially if log is not complete
    VALGRIND_CHECK_1=$(grep "==[0-9]\+== ERROR SUMMARY: [0-9]\+ error" ${RUNDIR}/${TRIAL}/log/master.err | sed 's|.*ERROR SUMMARY: \([0-9]\+\) error.*|\1|')
    if [ "${VALGRIND_CHECK_1}" == "" ]; then VALGRIND_CHECK_1=0; fi
    if [ ${VALGRIND_CHECK_1} -gt 0 ]; then 
      VALGRIND_ERRORS_FOUND=1; 
    fi
    if egrep -qi "^[ \t]*==[0-9]+[= \t]+[atby]+[ \t]*0x" ${RUNDIR}/${TRIAL}/log/master.err; then 
      VALGRIND_ERRORS_FOUND=1; 
    fi
    if egrep -qi "==[0-9]+== ERROR SUMMARY: [1-9]" ${RUNDIR}/${TRIAL}/log/master.err; then 
      VALGRIND_ERRORS_FOUND=1; 
    fi
    if [ ${VALGRIND_ERRORS_FOUND} -eq 1 ]; then
      VALGRIND_TEXT=`${SCRIPT_PWD}/valgrind_string.sh ${RUNDIR}/${TRIAL}/log/master.err`
      echoit "Valgrind error detected: ${VALGRIND_TEXT}"
      if [ ${TRIAL_SAVED} -eq 0 ]; then
        savetrial
        TRIAL_SAVED=1
      fi
    else
      # Report that no Valgrnid errors were found & Include ERROR SUMMARY from error log
      echoit "No Valgrind errors detected. $(grep "==[0-9]\+== ERROR SUMMARY: [0-9]\+ error" ${RUNDIR}/${TRIAL}/log/master.err | sed 's|.*ERROR S|ERROR S|')"
    fi
  fi
  if [ $(ls -l ${RUNDIR}/${TRIAL}/*/*core.* 2>/dev/null | wc -l) -ge 1 ]; then
    echoit "mysqld coredump detected at $(ls ${RUNDIR}/${TRIAL}/*/*core.*)"
    if [ ${PXC} -eq 0 ]; then
      echoit "Bug found (as per error log): `${SCRIPT_PWD}/text_string.sh ${RUNDIR}/${TRIAL}/log/master.err`"
    else
      CORE1=`ls ${RUNDIR}/${TRIAL}/1/*core.* 2>/dev/null`
      CORE2=`ls ${RUNDIR}/${TRIAL}/2/*core.* 2>/dev/null`
      CORE3=`ls ${RUNDIR}/${TRIAL}/3/*core.* 2>/dev/null`
      if [ ! "${CORE1}" == "" ]; then echoit "Bug found in PXC node #1 (as per error log): `${SCRIPT_PWD}/text_string.sh ${CORE1}`"; fi
      if [ ! "${CORE2}" == "" ]; then echoit "Bug found in PXC node #2 (as per error log): `${SCRIPT_PWD}/text_string.sh ${CORE2}`"; fi
      if [ ! "${CORE3}" == "" ]; then echoit "Bug found in PXC node #3 (as per error log): `${SCRIPT_PWD}/text_string.sh ${CORE3}`"; fi
    fi
    if [ ${TRIAL_SAVED} -eq 0 ]; then
      savetrial
      TRIAL_SAVED=1
    fi
  elif [ ${SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY} -eq 0 ]; then
    if [ ${TRIAL_SAVED} -eq 0 ]; then
      echoit "Saving full trial outcome (as SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY=0 and so trials are saved irrespective of whetter an issue was detected or not)"
      savetrial
      TRIAL_SAVED=1
    fi
  else
    if [ ${SAVE_SQL} -eq 1 ]; then 
      if [ ${VALGRIND_RUN} -eq 1 ]; then
        if [ ${VALGRIND_ERRORS_FOUND} -ne 1 ]; then
          echoit "Not saving anything for this trial (as SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY=1, and no coredump or Valgrind issue was generated), except the SQL trace (as SAVE_SQL=1)"
        fi
      else
        echoit "Not saving anything for this trial (as SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY=1, and no coredump was generated), except the SQL trace (as SAVE_SQL=1)"
      fi
      savesql
    else
      if [ ${VALGRIND_RUN} -eq 1 ]; then
        if [ ${VALGRIND_ERRORS_FOUND} -ne 1 ]; then
          echoit "Not saving anything for this trial (as SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY=1 as well as SAVE_SQL=0, and no coredump or Valgrind issue was generated)" 
        fi
      else
        echoit "Not saving anything for this trial (as SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY=1 as well as SAVE_SQL=0, and no coredump was generated)" 
      fi
    fi
  fi  
}

# Setup
rm -Rf ${WORKDIR} ${RUNDIR}
mkdir ${WORKDIR} ${WORKDIR}/log ${RUNDIR}
WORKDIRACTIVE=1
# User for recovery testing
echo "create user recovery@'%';grant all on *.* to recovery@'%';flush privileges;" > ${WORKDIR}/recovery-user.sql
if [ ${PXC} -eq 0 ];then
  echoit "Workdir: ${WORKDIR} | Rundir: ${RUNDIR} | Basedir: ${BASEDIR}"
else
  echoit "Workdir: ${WORKDIR} | Rundir: ${RUNDIR} | Basedir: ${BASEDIR} | PXC Docker Mode: Active"
fi
echoit "mysqld Start Timeout: ${MYSQLD_START_TIMEOUT} | Client Threads: ${THREADS} | Queries/Thread: ${QUERIES_PER_THREAD} | Trials: ${TRIALS} | Save coredump/valgrind issue trials only: `if [ ${SAVE_TRIALS_WITH_CORE_OR_VALGRIND_ONLY} -eq 1 ]; then echo -n 'TRUE'; if [ ${SAVE_SQL} -eq 1 ]; then echo ' + save all SQL traces'; else echo ''; fi; else echo 'FALSE'; fi`"
echoit "Valgrind run: `if [ ${VALGRIND_RUN} -eq 1 ]; then echo -n 'TRUE'; else echo -n 'FALSE'; fi` | pquery timeout: ${PQUERY_RUN_TIMEOUT} | SQL file used: ${INFILE} `if [ ${THREADS} -ne 1 ]; then echo -n "| Testcase size (chunked from infile): ${MULTI_THREADED_TESTC_LINES}"; fi`"
echoit "pquery Binary: ${PQUERY_BIN}"
if [ "${MYEXTRA}" != "" ]; then echoit "MYEXTRA: ${MYEXTRA}"; fi
if [ "${MYSAFE}" != "" ]; then echoit "MYSAFE: ${MYSAFE}"; fi
echoit "Making a copy of the pquery binary used (${PQUERY_BIN}) to ${WORKDIR}/ (handy for later re-runs/reference etc.)"
cp ${PQUERY_BIN} ${WORKDIR}
echoit "Making a copy of this script (${SCRIPT}) to ${WORKDIR}/ for reference & adding pquery- prefix (avoids pquery-prep-run not finding the script)..."  # pquery- prefix avoids pquer-prep-red.sh script-locating issues if this script had been renamed to a name without 'pquery' in it.
cp ${SCRIPT_AND_PATH} ${WORKDIR}/pquery-${SCRIPT}
if [ ${STORE_COPY_OF_INFILE} -eq 1 ]; then
  echoit "Making a copy of the SQL input file used (${INFILE}) to ${WORKDIR}/ for reference..."
  cp ${INFILE} ${WORKDIR}
fi

if [ "$(${BIN} --version | grep -oe '5\.[1567]' | head -n1)" == "5.7" ]; then
  VERSION_CHK=`${BIN}  --version | grep -oe '5\.[1567]\.[0-9]*' | cut -f3 -d'.' | head -n1`
  if [[ $VERSION_CHK -ge 5 ]]; then
    MID_OPT="--no-defaults --initialize-insecure"
  else
    MID_OPT="--insecure"
  fi
else
  MID_OPT="--force --no-defaults"
fi

if [ "${MID_OPT}" == "--no-defaults --initialize-insecure" ]; then
    MID_57="${BASEDIR}/bin/mysqld"
  else
    MID_57="${BASEDIR}/bin/mysql_install_db"
  fi

if [ ${PXC} -eq 0 ]; then
  echoit "Making a copy of the mysqld used to ${WORKDIR}/mysqld (handy for coredump analysis and manual bundle creation)..."  # Fig setup also does this for PXC elsewhere
  mkdir ${WORKDIR}/mysqld
  cp ${BIN} ${WORKDIR}/mysqld
  echoit "Making a copy of the ldd files required for mysqld core analysis to ${WORKDIR}/mysqld..."
  PWDTMPSAVE=$PWD
  cd ${WORKDIR}/mysqld
  ${SCRIPT_PWD}/ldd_files.sh  # TODO for PXC, not sure yet how
  cd ${PWDTMPSAVE}
  echoit "Generating datadir template (using mysql_install_db or mysqld --init)..."
  if [ -r ${BASEDIR}/bin/mysql_install_db ]; then 
    $MID_57 $MID_OPT --basedir=${BASEDIR} --datadir=${WORKDIR}/data.template > ${WORKDIR}/log/mysql_install_db.txt 2>&1
  elif [ -r ${BASEDIR}/scripts/mysql_install_db ]; then 
    ${BASEDIR}/scripts/mysql_install_db $MID_OPT --basedir=${BASEDIR} --datadir=${WORKDIR}/data.template > ${WORKDIR}/log/mysql_install_db.txt 2>&1
  else 
    echoit "Error: mysql_install_db not found in $PWD/scripts nor in $PWD/bin"
    exit 1
  fi
  # Sysbench dataload
  if [ ${SYSBENCH_DATALOAD} -eq 1 ]; then
    echoit "Starting mysqld for sysbench data load. Error log is stored at ${WORKDIR}/data.template/master.err"
    CMD="${BIN} --basedir=${BASEDIR} --datadir=${WORKDIR}/data.template --tmpdir=${WORKDIR}/data.template \
      --core-file --port=$PORT --pid_file=${WORKDIR}/data.template/pid.pid --socket=${WORKDIR}/data.template/socket.sock \
      --log-output=none --log-error=${WORKDIR}/data.template/master.err"

    $CMD > ${WORKDIR}/data.template/master.err 2>&1 &
    MPID="$!"

    for X in $(seq 0 ${MYSQLD_START_TIMEOUT}); do
      sleep 1
      if ${BASEDIR}/bin/mysqladmin -uroot -S${WORKDIR}/data.template/socket.sock ping > /dev/null 2>&1; then
        break
      fi
      if [ "${MPID}" == "" ]; then
        echoit "Assert! ${MPID} empty. Terminating!"
        exit 1
      fi
    done
    # Sysbench run for data load
    /usr/bin/sysbench --test=${SCRIPT_PWD}/sysbench_scripts/parallel_prepare.lua --num-threads=1 --oltp-tables-count=1 --oltp-table-size=1000000 --mysql-db=test --mysql-user=root --db-driver=mysql --mysql-socket=${WORKDIR}/data.template/socket.sock run > ${WORKDIR}/data.template/sysbench_prepare.txt 2>&1

    # Terminate mysqld
    timeout --signal=9 20s ${BASEDIR}/bin/mysqladmin -uroot -S${WORKDIR}/data.template/socket.sock shutdown > /dev/null 2>&1
    (sleep 0.2; kill -9 ${MPID} >/dev/null 2>&1; wait ${MPID} >/dev/null 2>&1) &  # Terminate mysqld
  fi
  echo "${MYEXTRA}${MYSAFE}" | if grep -qi "innodb[_-]log[_-]checksum[_-]algorithm"; then
    # Ensure that if MID created log files with the standard checksum algo, whilst we start the server with another one, that log files are re-created by mysqld
    rm ${WORKDIR}/data.template/ib_log*
  fi
else
  echoit "Ensuring that PXC Docker Images are ready to go..."
  if [ $(sudo docker images | grep "pqueryjenkins_pxc" | wc -l) -ne 3 ]; then
    echoit "Assert: $(sudo docker images | grep "new_pxc" | wc -l) != 3"
    echoit "Did you run cd ${DOCKER_COMPOSE_LOC}; sudo docker-compose up followed by CTRL+C when the cluster was fully up (takes about 5 minutes) ? Terminating."
    exit 1
  else
    echoit "Found 3 dockercompose_pxc images in place/ready to go, proceeding..."
  fi
fi

# Start actual pquery testing
echoit "Starting pquery testing iterations..."
COUNT=0
for X in $(seq 1 ${TRIALS}); do
  pquery_test
  COUNT=$[ $COUNT + 1 ]
done
# All done, wrap up pquery run
echoit "pquery finished requested number of trials (${TRIALS})... Terminating..."
if [ ${PXC} -eq 0 ]; then 
  echoit "Cleaning up any leftover processes..."
  KILL_PIDS=`ps -ef | grep "$RANDOMD" | grep -v "grep" | awk '{print $2}' | tr '\n' ' '`
  if [ "${KILL_PIDS}" != "" ]; then
    echoit "Terminating the following PID's: ${KILL_PIDS}"
    kill -9 ${KILL_PIDS} >/dev/null 2>&1
  fi
else
  echoit "Cleaning up remaining Docker containers..."
  ${DOCKER_COMPOSE_LOC}/cleanup.sh
fi
echoit "Done. Attempting to cleanup the pquery rundir ${RUNDIR}..."
rm -Rf ${RUNDIR}
echoit "The results of this run can be found in the workdir ${WORKDIR}..."
echoit "Done. Exiting $0 with exit code 0..."
exit 0