Script changes and folder structure, a folder for TLA+ spec is also a…

…dded

bin/copy-hermes-files.sh → bin/copy-exec-files.sh

@@ -13,24 +13,23 @@ HOSTS=( ##### network  cluster #####
         )
 
 FILES=(
-        "run-hermes.sh"
-        "hermes"
+        "run-hermesKV.sh"
+        "hermesKV"
+        "run-rCRAQ.sh"
+        "rCRAQ"
       )
 
 USERNAME="s1671850" # "user"
 LOCAL_HOST=`hostname`
-MAKE_FOLDER="/home/${USERNAME}/hermes/src"
-HOME_FOLDER="/home/${USERNAME}/hermes/src/hermes"
-DEST_FOLDER="/home/${USERNAME}/hermes-exec/src/hermes"
+EXEC_FOLDER="/home/${USERNAME}/hermes/exec"
 
-cd $MAKE_FOLDER
-make clean
-make
+cd $EXEC_FOLDER
+make clean; make
 cd -
 
 for FILE in "${FILES[@]}"
 do
-	parallel scp ${HOME_FOLDER}/${FILE} {}:${DEST_FOLDER}/${FILE} ::: $(echo ${HOSTS[@]/$LOCAL_HOST})
+	parallel scp ${EXEC_FOLDER}/${FILE} {}:${EXEC_FOLDER}/${FILE} ::: $(echo ${HOSTS[@]/$LOCAL_HOST})
 	echo "${FILE} copied to {${HOSTS[@]/$LOCAL_HOST}}"
 done
 

bin/copy-n-exec-cr-files.sh

@@ -13,8 +13,8 @@ HOSTS=( ##### network  cluster #####
         )
 
 FILES=(
-        "run-cr.sh"
-        "cr"
+        "run-rCRAQ.sh"
+        "rCRAQ"
       )
 
 USE_SAME_BATCH_N_CREDITS=1
@@ -41,26 +41,9 @@ LAT_WORKER="-1"
 
 USERNAME="s1671850" # "user"
 LOCAL_HOST=`hostname`
-MAKE_FOLDER="/home/${USERNAME}/hermes/src"
-HOME_FOLDER="/home/${USERNAME}/hermes/src/CR"
-DEST_FOLDER="/home/${USERNAME}/hermes-exec/src/CR"
-RESULT_FOLDER="/home/${USERNAME}/hermes-exec/results/xput/per-node/"
-RESULT_OUT_FOLDER="/home/${USERNAME}/hermes/results/xput/per-node/"
-RESULT_OUT_FOLDER_MERGE="/home/${USERNAME}/hermes/results/xput/all-nodes/"
 
-cd $MAKE_FOLDER
-make clean
-make
-cd -
-
-for FILE in "${FILES[@]}"
-do
-	parallel scp ${HOME_FOLDER}/${FILE} {}:${DEST_FOLDER}/${FILE} ::: $(echo ${HOSTS[@]/$LOCAL_HOST})
-	echo "${FILE} copied to {${HOSTS[@]/$LOCAL_HOST}}"
-done
-
-REMOTE_COMMAND="cd ${DEST_FOLDER}; bash run-cr.sh"
-OUTP_FOLDER="/home/${USERNAME}/hermes/results/xput/per-node/"
+EXEC_FOLDER="/home/${USERNAME}/hermes/exec"
+REMOTE_COMMAND="cd ${EXEC_FOLDER}; bash run-rCRAQ.sh"
 
 PASS="${1}"
 if [ -z "$PASS" ]
@@ -70,7 +53,9 @@ then
 fi
 
 echo "\$PASS is OK!"
-cd ${HOME_FOLDER}
+cd ${EXEC_FOLDER}
+
+../bin/copy-exec-files.sh
 
 if [ ${USE_SAME_BATCH_N_CREDITS} -eq 0 ]
 then
@@ -110,19 +95,4 @@ else
    done
 fi
 
-# Gather remote files
-parallel "scp {}:${RESULT_FOLDER}* ${RESULT_OUT_FOLDER} " ::: $(echo ${HOSTS[@]/$LOCAL_HOST})
-echo "xPut result files copied from: {${HOSTS[@]/$LOCAL_HOST}}"
-
-	  # group all files
-ls ${RESULT_OUT_FOLDER} | awk -F '-' '!x[$1]++{print $1}' | while read -r line; do
-        #// Create an intermediate file print the 3rd line for all files with the same prefix to the same file
-    awk 'FNR==3 {print $0}' ${RESULT_OUT_FOLDER}/$line* > ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt
-    #   Sum up the xPut of the (3rd iteration) from every node to create the final file
-    awk -F ':' '{sum += $2} END {print sum}' ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt > ${RESULT_OUT_FOLDER_MERGE}/$line.txt
-    rm -rf  ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt
-done
-
-cd -
-
-
+../bin/get-system-xput-files.sh

bin/copy-n-exec-hermes-files.sh

@@ -5,105 +5,73 @@ HOSTS=( ##### network  cluster #####
          "austin"
          "indianapolis"
          "philly"
-#         "atlanta"
+         "atlanta"
          ##### compute cluster #####
 #         "baltimore"
 #         "chicago"
 #         "detroit"
         )
 
 FILES=(
-        "run-hermes.sh"
-        "hermes"
+        "run-hermesKV.sh"
+        "hermesKV"
       )
 
 
 ### Runs to make
 #declare -a write_ratios=(0 10 50 200 500 1000)
-#declare -a write_ratios=(500 750 1000)
-#declare -a write_ratios=(200)
-#declare -a write_ratios=(10 20 50)
 declare -a write_ratios=(1000)
 declare -a rmw_ratios=(0)
-#declare -a rmw_ratios=(1000)
 #declare -a num_workers=(5 10 15 20 25 30 36)
 declare -a num_workers=(1)
 #declare -a batch_sizes=(25 50 75 100 125 150 200 250)
 declare -a batch_sizes=(50)
 declare -a credits=(15)
-#declare -a credits=(30)
 #declare -a coalesce=(1 5 10 15)
 declare -a coalesce=(15)
+#declare -a num_machines=(2 3 5 7)
 declare -a num_machines=(2)
+
 # Set LAT_WORKER to -1 to disable latency measurement or to worker id (i.e., from 0 up to [num-worker - 1])
 LAT_WORKER="-1"
 #LAT_WORKER="0"
 
 USERNAME="s1671850" # "user"
 LOCAL_HOST=`hostname`
-MAKE_FOLDER="/home/${USERNAME}/hermes/src"
-HOME_FOLDER="/home/${USERNAME}/hermes/src/hermes"
-DEST_FOLDER="/home/${USERNAME}/hermes-exec/src/hermes"
-RESULT_FOLDER="/home/${USERNAME}/hermes-exec/results/xput/per-node/"
-RESULT_OUT_FOLDER="/home/${USERNAME}/hermes/results/xput/per-node/"
-RESULT_OUT_FOLDER_MERGE="/home/${USERNAME}/hermes/results/xput/all-nodes/"
-
-cd $MAKE_FOLDER
-make clean
-make
-cd -
-
-for FILE in "${FILES[@]}"
-do
-	parallel scp ${HOME_FOLDER}/${FILE} {}:${DEST_FOLDER}/${FILE} ::: $(echo ${HOSTS[@]/$LOCAL_HOST})
-	echo "${FILE} copied to {${HOSTS[@]/$LOCAL_HOST}}"
-done
+EXEC_FOLDER="/home/${USERNAME}/hermes/exec"
 
-REMOTE_COMMAND="cd ${DEST_FOLDER}; bash run-hermes.sh"
-OUTP_FOLDER="/home/${USERNAME}/hermes/results/xput/per-node/"
+REMOTE_COMMAND="cd ${EXEC_FOLDER}; bash run-hermesKV.sh"
 
 PASS="${1}"
 if [ -z "$PASS" ]
 then
       echo "\$PASS is empty! --> sudo pass for remotes is expected to be the first arg"
-else
-      echo "\$PASS is OK!"
-      cd ${HOME_FOLDER}
+      exit;
+fi
+
+echo "\$PASS is OK!"
+cd ${EXEC_FOLDER}
+
+../bin/copy-exec-files.sh
 
       # Execute locally and remotely
-  for M in "${num_machines[@]}"; do
+for M in "${num_machines[@]}"; do
     for RMW in "${rmw_ratios[@]}"; do
       for WR in "${write_ratios[@]}"; do
         for W in "${num_workers[@]}"; do
           for BA in "${batch_sizes[@]}"; do
             for CRD in "${credits[@]}"; do
               for COAL in "${coalesce[@]}"; do
                  args=" -M ${M} -R ${RMW} -w ${WR} -W ${W} -b ${BA} -c ${CRD} -C ${COAL} -l ${LAT_WORKER}"
-                 echo ${PASS} | ./run-hermes.sh ${args} &
-                 sleep 2
+                 echo ${PASS} | ./run-hermesKV.sh ${args} &
+                 sleep 2 # give some leeway so that manager starts before executing the members
 	             parallel "echo ${PASS} | ssh -tt {} $'${REMOTE_COMMAND} ${args}'" ::: $(echo ${HOSTS[@]/$LOCAL_HOST}) >/dev/null
 	          done
 	        done
 	      done
 	    done
 	  done
 	done
-  done
-
-      # Gather remote files
-	  parallel "scp {}:${RESULT_FOLDER}* ${RESULT_OUT_FOLDER} " ::: $(echo ${HOSTS[@]/$LOCAL_HOST})
-	  echo "xPut result files copied from: {${HOSTS[@]/$LOCAL_HOST}}"
-
-	  # group all files
-      ls ${RESULT_OUT_FOLDER} | awk -F '-' '!x[$2]++{print $1}' | while read -r line; do
-          #// Create an intermediate file print the 3rd line for all files with the same prefix to the same file
-          awk 'FNR==3 {print $0}' ${RESULT_OUT_FOLDER}/$line* > ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt
-          #   Sum up the xPut of the (3rd iteration) from every node to create the final file
-          awk -F ':' '{sum += $2} END {print sum}' ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt > ${RESULT_OUT_FOLDER_MERGE}/$line.txt
-          rm -rf  ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt
-      done
-
-	  cd -
-fi
-
+done
 
+../bin/get-system-xput-files.sh

bin/get-system-xput-files.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+HOSTS=( ##### network  cluster #####
+         "houston"
+         "sanantonio"
+         "austin"
+         "indianapolis"
+         "philly"
+         "atlanta"
+         ##### compute cluster #####
+#         "baltimore"
+#         "chicago"
+#         "detroit"
+        )
+
+USERNAME="s1671850" # "user"
+LOCAL_HOST=`hostname`
+EXEC_FOLDER="/home/${USERNAME}/hermes/exec"
+RESULTS_FOLDER="/home/${USERNAME}/hermes/exec/results"
+
+RESULT_FOLDER="${RESULTS_FOLDER}/xput/per-node/"
+RESULT_OUT_FOLDER="${RESULTS_FOLDER}/xput/per-node/"
+RESULT_OUT_FOLDER_MERGE="${RESULTS_FOLDER}/xput/all-nodes/"
+
+# Gather remote files
+parallel "scp {}:${RESULT_FOLDER}* ${RESULT_OUT_FOLDER} " ::: $(echo ${HOSTS[@]/$LOCAL_HOST})
+echo "xPut result files copied from: {${HOSTS[@]/$LOCAL_HOST}}"
+
+# group all files
+ls ${RESULT_OUT_FOLDER} | awk -F '-' '!x[$2]++{print $1}' | while read -r line; do
+    # Create an intermediate file print the 3rd line for all files with the same prefix to the same file
+    awk 'FNR==3 {print $0}' ${RESULT_OUT_FOLDER}/$line* > ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt
+          #   Sum up the xPut of the (3rd iteration) from every node to create the final file
+    awk -F ':' '{sum += $2} END {print sum}' ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt > ${RESULT_OUT_FOLDER_MERGE}/$line.txt
+    rm -rf  ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt
+done
+
+echo "System-wide xPut results produced in ${RESULT_OUT_FOLDER_MERGE} directory!"

exec/Makefile

@@ -0,0 +1,35 @@
+CPPFLAGS  := -O3 #-Wno-unused-result -Wall -Werror
+LD      := gcc -O3 -flto
+LDFLAGS := ${LDFLAGS} -libverbs -lrt -lpthread -lmemcached -lnuma -lrdmacm
+CFLAGS   =  -I../include/mica-herd -I../include/hermes -I../include/wings -I../include/hades
+APPS    := hermesKV rCRAQ
+PROF    := -g -fno-omit-frame-pointer
+
+all: ${APPS} clean-o
+
+hermesKV: ../src/wings/wings.o ../src/hades/hades.o \
+          ../src/mica-herd/herd.o ../src/mica-herd/mica.o ../src/mica-herd/city.o \
+          ../src/hermes/main.o ../src/hermes/hermes_worker.o ../src/hermes/util.o \
+          ../src/hermes/stats.o ../src/hermes/spacetime.o ../src/hermes/hermesKV.o
+	${LD} -o $@ $^ ${LDFLAGS}
+
+
+rCRAQ: ../src/mica-herd/herd.o ../src/mica-herd/mica.o \
+       ../src/mica-herd/city.o ../src/hermes/main.o ../src/CR/cr_worker.o ../src/CR/crKV.o \
+       ../src/hermes/spacetime.o ../src/hermes/util.o ../src/hermes/stats.o  ../src/wings/wings.o
+	${LD} -o $@ $^ ${LDFLAGS}
+
+
+hades-exec: ../src/hades/hades.o ../src/hades/test.o ../src/wings/wings.o ../src/mica-herd/herd.o
+	${LD} -o hades $^ ${LDFLAGS}
+
+hades: hades-exec clean-o
+
+PHONY: clean
+clean:
+	@rm -f ../src/hermes/*.o ../src/mica-herd/*.o ../src/wings/*.o \
+	      ../src/CR/*.o ../src/hades/*.o ${APPS} hades
+
+clean-o:
+	@rm -f ../src/hermes/*.o ../src/mica-herd/*.o ../src/wings/*.o \
+	      ../src/CR/*.o ../src/hades/*.o