Updated HSM version

mcloudlabs · Sep 7, 2021 · a1c4e9a · a1c4e9a
1 parent 8a84020
commit a1c4e9a
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 76 deletions.
diff --git a/examples/lustre_avset/config.json b/examples/lustre_avset/config.json
@@ -21,7 +21,7 @@
         "lustre_instances": 4,
         "low_priority": false,
         "storage_account": "<NOT-SET>",
-        "storage_key": "sakey.{{variables.storage_account}}",
+        "storage_sas": "saskey.{{variables.storage_account}}.{{variables.storage_container}},rwdl,1y",
         "storage_container": "<NOT-SET>",
         "log_analytics_lfs_name": "<NOT-SET>",
         "la_resourcegroup": "<NOT-SET>",
@@ -182,9 +182,8 @@
             "args": [
                 "lustremaster",
                 "variables.storage_account",
-                "variables.storage_key",
-                "variables.storage_container",
-                "variables.lustre_version"
+                "variables.storage_sas",
+                "variables.storage_container"
             ],
             "tag": "lustre",
             "sudo": true
@@ -202,10 +201,9 @@
             "script": "lfsimport.sh",
             "args": [
                 "variables.storage_account",
-                "variables.storage_key",
+                "variables.storage_sas",
                 "variables.storage_container",
-                "variables.lustre_mount",
-                "variables.lustre_version"
+                "variables.lustre_mount"
             ],
             "tag": "lfsazimport",
             "sudo": true

diff --git a/examples/lustre_combined/config.json b/examples/lustre_combined/config.json
@@ -21,7 +21,7 @@
         "lustre_instances": 4,
         "low_priority": false,
         "storage_account": "<NOT-SET>",
-        "storage_key": "sakey.{{variables.storage_account}}",
+        "storage_sas": "saskey.{{variables.storage_account}}.{{variables.storage_container}},rwdl,1y",
         "storage_container": "<NOT-SET>",
         "log_analytics_lfs_name": "<NOT-SET>",
         "la_resourcegroup": "<NOT-SET>",
@@ -163,9 +163,8 @@
             "args": [
                 "$(head -n1 hostlists/tags/lustre)",
                 "variables.storage_account",
-                "variables.storage_key",
-                "variables.storage_container",
-                "variables.lustre_version"
+                "variables.storage_sas",
+                "variables.storage_container"
             ],
             "tag": "lustre",
             "sudo": true
@@ -183,10 +182,9 @@
             "script": "lfsimport.sh",
             "args": [
                 "variables.storage_account",
-                "variables.storage_key",
+                "variables.storage_sas",
                 "variables.storage_container",
-                "variables.lustre_mount",
-                "variables.lustre_version"
+                "variables.lustre_mount"
             ],
             "tag": "lfsazimport",
             "sudo": true

diff --git a/examples/lustre_full/config.json b/examples/lustre_full/config.json
@@ -20,7 +20,7 @@
         "compute_instances": 2,
         "oss_instances": 2,
         "storage_account": "<NOT-SET>",
-        "storage_key": "sakey.{{variables.storage_account}}",
+        "storage_sas": "saskey.{{variables.storage_account}}.{{variables.storage_container}},rwdl,1y",
         "storage_container": "<NOT-SET>",
         "la_resourcegroup": "<NOT-SET>",
         "la_name": "<NOT-SET>",
@@ -194,9 +194,8 @@
             "args": [
                 "$(<hostlists/tags/lfsmaster)",
                 "variables.storage_account",
-                "variables.storage_key",
-                "variables.storage_container",
-                "variables.lustre_version"
+                "variables.storage_sas",
+                "variables.storage_container"
             ],
             "tag": "lfshsm",
             "sudo": true
@@ -219,10 +218,9 @@
             "script": "lfsimport.sh",
             "args": [
                 "variables.storage_account",
-                "variables.storage_key",
+                "variables.storage_sas",
                 "variables.storage_container",
-                "variables.lustre_mount",
-                "variables.lustre_version"
+                "variables.lustre_mount"
             ],
             "tag": "lfsazimport",
             "sudo": true

diff --git a/examples/lustre_manageddisk/config.json b/examples/lustre_manageddisk/config.json
@@ -19,7 +19,7 @@
     "compute_instances": 2,
     "oss_instances": 2,
     "storage_account": "<NOT-SET>",
-    "storage_key": "sakey.{{variables.storage_account}}",
+    "storage_sas": "saskey.{{variables.storage_account}}.{{variables.storage_container}},rwdl,1y",
     "storage_container": "<NOT-SET>",
     "log_analytics_lfs_name": "<NOT-SET>",
     "la_resourcegroup": "<NOT-SET>",
@@ -205,11 +205,10 @@
     {
       "script": "lfshsm.sh",
       "args": [
-        "$(<hostlists/tags/lfsmaster)",
+        "$(head -n1 hostlists/tags/lustre)",
         "variables.storage_account",
-        "variables.storage_key",
-        "variables.storage_container",
-        "variables.lustre_version"
+        "variables.storage_sas",
+        "variables.storage_container"
       ],
       "tag": "lfshsm",
       "sudo": true
@@ -232,10 +231,9 @@
       "script": "lfsimport.sh",
       "args": [
         "variables.storage_account",
-        "variables.storage_key",
+        "variables.storage_sas",
         "variables.storage_container",
-        "variables.lustre_mount",
-        "variables.lustre_version"
+        "variables.lustre_mount"
       ],
       "tag": "lfsazimport",
       "sudo": true

diff --git a/scripts/lfshsm.sh b/scripts/lfshsm.sh
@@ -2,31 +2,18 @@
 
 # arg: $1 = lfsserver
 # arg: $2 = storage account
-# arg: $3 = storage key
+# arg: $3 = storage sas
 # arg: $4 = storage container
-# arg: $5 = lustre version (default 2.10)
 master=$1
 storage_account=$2
-storage_key=$3
+storage_sas=$3
 storage_container=$4
-lustre_version=${5-2.10}
 
-# remove the patch version
-ndots=${lustre_version//[^.]} 
-if [ "${#ndots}" = "2" ]; then
-    lustre_version=${lustre_version%.*}
-fi
-
-# adding kernel module for lustre client
-if [ "$lustre_version" = "2.10" ]; then
-    yum install -y kmod-lustre-client
-    weak-modules --add-kernel $(uname -r)
-fi
 
 if ! rpm -q lemur-azure-hsm-agent lemur-azure-data-movers; then
     yum -y install \
-        https://azurehpc.azureedge.net/rpms/lemur-azure-hsm-agent-1.0.0-lustre_${lustre_version}.x86_64.rpm \
-        https://azurehpc.azureedge.net/rpms/lemur-azure-data-movers-1.0.0-lustre_${lustre_version}.x86_64.rpm
+        https://azurehpc.azureedge.net/rpms/lemur-azure-hsm-agent-2.0.0-lustre_2.12.x86_64.rpm \
+        https://azurehpc.azureedge.net/rpms/lemur-azure-data-movers-2.0.0-lustre_2.12.x86_64.rpm
 fi
 
 mkdir -p /var/run/lhsmd
@@ -56,19 +43,20 @@ EOF
 chmod 600 /etc/lhsmd/agent
 
 cat <<EOF >/etc/lhsmd/lhsm-plugin-az
-az_storage_account = "$storage_account"
-az_storage_key = "$storage_key"
-
-num_threads = 32
-
-#
-# One or more archive definition is required.
-#
-archive  "az-blob" {
-    id = 1                           # Must be unique to this endpoint
-    container = "$storage_container" # Container used for this archive
-    prefix = ""                   # Optional prefix
-    num_threads = 32
+num_threads=16
+az_storage_account="$storage_account"
+az_storage_sas="?$storage_sas"
+az_kv_name=""
+az_kv_secret_name=""
+region="westeurope"
+bandwidth=0
+exportprefix=""
+archive "archive1" {
+    id=1
+    num_threads=15
+    root=""
+    compression="off"
+    container="$storage_container"
 }
 EOF
 chmod 600 /etc/lhsmd/lhsm-plugin-az

diff --git a/scripts/lfsimport.sh b/scripts/lfsimport.sh
@@ -1,31 +1,22 @@
 #!/bin/bash
 
 # arg: $1 = storage account
-# arg: $2 = storage key
+# arg: $2 = storage sas
 # arg: $3 = storage container
 # arg: $3 = lfs mount
 # arg: $4 = lustre mount (default=/lustre)
-# arg: $5 = lustre version (default=2.10)
-storage_account=$1
-storage_key=$2
-storage_container=$3
+storage_account="$1"
+storage_sas="$2"
+storage_container="$3"
 lfs_mount=${4:-/lustre}
-lustre_version=${5-2.10}
-
-# remove the patch version
-ndots=${lustre_version//[^.]} 
-if [ "${#ndots}" = "2" ]; then
-    lustre_version=${lustre_version%.*}
-fi
 
 if ! rpm -q lemur-azure-hsm-agent lemur-azure-data-movers; then
     yum -y install \
-        https://azurehpc.azureedge.net/rpms/lemur-azure-hsm-agent-1.0.0-lustre_${lustre_version}.x86_64.rpm \
-        https://azurehpc.azureedge.net/rpms/lemur-azure-data-movers-1.0.0-lustre_${lustre_version}.x86_64.rpm
+        https://azurehpc.azureedge.net/rpms/lemur-azure-hsm-agent-2.0.0-lustre_2.12.x86_64.rpm \
+        https://azurehpc.azureedge.net/rpms/lemur-azure-data-movers-2.0.0-lustre_2.12.x86_64.rpm
 fi
 
 cd $lfs_mount
-export STORAGE_ACCOUNT=$storage_account
-export STORAGE_KEY=$storage_key
-/sbin/azure-import ${storage_container}
+export STORAGE_SAS="?$storage_sas"
+/sbin/azure-import -account ${storage_account} -container ${storage_container}