Added RAID health check test

experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/azure_raid_health.nhc

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Check for the health of a RAID array of a specified size associated to a given mount point
+
+function check_raid()
+{
+    MOUNT_PATH=$1
+    ARRAY_SIZE=$2
+
+    MOUNT=$(grep "${MOUNT_PATH} " /proc/mounts)
+    [ $? != 0 ] && die 1 "${MOUNT_PATH} not a valid mount point" && return 1
+
+    DEVICE=$(echo ${MOUNT} | awk '{print $1}')
+    dbg "Found device $DEVICE mounted at ${MOUNT_PATH}"
+
+    mdadm --detail ${DEVICE} > /dev/null
+    [ $? != 0 ] && die 1 "Device ${DEVICE} does not appear to be a RAID array" && return 1
+
+    RAID_STATE=$(mdadm --detail ${DEVICE} | awk '/State :/ {print $3}')
+    if [ "${RAID_STATE}" == 'clean' ]; then
+        dbg "RAID device ${DEVICE} is clean"
+    else
+        die 1 "RAID device ${DEVICE} is not in clean state (${RAID_STATE})"
+        return 1
+    fi
+    RAID_DEVICES=$(sudo mdadm --detail ${DEVICE} | awk '/Working Devices :/ {print $4}')
+    if [ "${RAID_DEVICES}" == ${ARRAY_SIZE} ]; then
+        dbg "Found ${RAID_DEVICES} RAID devices in array ${DEVICE} "
+    else
+            die 1 "RAID device ${DEVICE} reported ${RAID_DEVICES} devices (expected ${ARRAY_SIZE})"
+        return 1
+    fi
+
+    TESTFILE=$(mktemp -p ${MOUNT_PATH} -t nhc_checkraid_XXXXXXXX.tmp)
+    if [ $? == 0 ]; then
+        dbg "File ${TESTFILE} successfully created"
+    else
+        die 1 "Cannot create file ${TESTFILE}"
+        return 1
+    fi
+
+    rm ${TESTFILE}
+    if [ $? == 0 ]; then
+        dbg "File ${TESTFILE} successfully deleted"
+    else
+        die 1 "Cannot delete file ${TESTFILE}"
+        return 1
+    fi
+
+    return 0
+}

experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/configure_nhc.sh

@@ -19,7 +19,7 @@ SLURM_HEALTH_CHECK_NODE_STATE=IDLE
 NHC_PROLOG=1
 NHC_EPILOG=0
 AUTOSCALING=0
-NHC_EXTRA_TEST_FILES="csc_nvidia_smi.nhc azure_cuda_bandwidth.nhc azure_gpu_app_clocks.nhc azure_gpu_ecc.nhc azure_gpu_persistence.nhc azure_ib_write_bw_gdr.nhc azure_nccl_allreduce_ib_loopback.nhc azure_ib_link_flapping.nhc azure_gpu_clock_throttling.nhc azure_cpu_drop_cache_mem.nhc azure_gpu_xid.nhc azure_nccl_allreduce.nhc"
+NHC_EXTRA_TEST_FILES="csc_nvidia_smi.nhc azure_cuda_bandwidth.nhc azure_gpu_app_clocks.nhc azure_gpu_ecc.nhc azure_gpu_persistence.nhc azure_ib_write_bw_gdr.nhc azure_nccl_allreduce_ib_loopback.nhc azure_ib_link_flapping.nhc azure_gpu_clock_throttling.nhc azure_cpu_drop_cache_mem.nhc azure_gpu_xid.nhc azure_nccl_allreduce.nhc azure_raid_health.nhc"
 
 source $CYCLECLOUD_SPEC_PATH/files/common_functions.sh
 

experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96amsr_v4.conf

@@ -86,7 +86,7 @@
  * || check_fs_iused /sched 100%
  * || check_fs_iused /shared 98%
  * || check_fs_iused /mnt/resource_nvme 98%
-
+ * || check_raid /mnt/resource_nvme 8
 
 #######################################################################
 ###

experimental/cc_slurm_nhc/cc_slurm_nhc/specs/default/cluster-init/files/nd96asr_v4.conf

@@ -84,7 +84,7 @@
  * || check_fs_iused /sched 100%
  * || check_fs_iused /shared 98%
  * || check_fs_iused /mnt/resource_nvme 98%
-
+ * || check_raid /mnt/resource_nvme 8
 
 #######################################################################
 ###