Add automated restart test infrastructure (solana-labs#17491)

multinode-demo/bootstrap-validator.sh

@@ -75,6 +75,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --maximum-snapshots-to-retain ]]; then
       args+=("$1" "$2")
       shift 2
+    elif [[ $1 == --no-snapshot-fetch ]]; then
+      args+=("$1")
+      shift
     else
       echo "Unknown argument: $1"
       $program --help

system-test/restart-testcases/restart_gce.yml

@@ -0,0 +1,17 @@
+steps:
+  - command: "system-test/testnet-automation.sh"
+    label: "Restart test on GCE"
+    env:
+      UPLOAD_RESULTS_TO_SLACK: "false"
+      CLOUD_PROVIDER: "gce"
+      ENABLE_GPU: "false"
+      NUMBER_OF_VALIDATOR_NODES: 4
+      VALIDATOR_NODE_MACHINE_TYPE: "--machine-type n1-standard-16"
+      NUMBER_OF_CLIENT_NODES: 1
+      ADDITIONAL_FLAGS: "--dedicated"
+      BOOTSTRAP_VALIDATOR_MAX_STAKE_THRESHOLD: 66
+      SKIP_PERF_RESULTS: "true"
+      TEST_TYPE: "script"
+      CUSTOM_SCRIPT: "system-test/restart-testcases/restart_test_automation.sh"
+    agents:
+      - "queue=gce-deploy"

system-test/restart-testcases/restart_test_automation.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# shellcheck disable=SC1090
+# shellcheck disable=SC1091
+source "$(dirname "$0")"/../automation_utils.sh
+
+RESULT_FILE="$1"
+
+startGpuMode="off"
+if [[ -z $ENABLE_GPU ]]; then
+  ENABLE_GPU=false
+fi
+if [[ "$ENABLE_GPU" = "true" ]]; then
+  startGpuMode="on"
+fi
+
+declare maybeAsyncNodeInit
+if [[ "$ASYNC_NODE_INIT" = "true" ]]; then
+  maybeAsyncNodeInit="--async-node-init"
+fi
+
+declare maybeExtraPrimordialStakes
+if [[ -n "$EXTRA_PRIMORDIAL_STAKES" ]]; then
+  maybeExtraPrimordialStakes="--extra-primordial-stakes $EXTRA_PRIMORDIAL_STAKES"
+fi
+
+# Restart the network
+"$REPO_ROOT"/net/net.sh stop
+
+sleep 2
+
+# shellcheck disable=SC2086
+"$REPO_ROOT"/net/net.sh start --skip-setup --no-snapshot-fetch --no-deploy \
+  --gpu-mode $startGpuMode $maybeAsyncNodeInit $maybeExtraPrimordialStakes
+
+# TODO add the test here
+
+echo "Restart Test Succeeded" >>"$RESULT_FILE"