Improve the k8s mount curve script

k8s/nbd/nbd-package/usr/bin/mount_curve_clouddisk.sh

@@ -31,41 +31,75 @@ function run() {
 
     # create CURVE volume
     echo "1. create CURVE volume"
-    volume=$(sudo curve_ops_tool list --fileName=/ | grep ${filename})
-    if [ ! "$volume" ]
+    # check volume exist in root dir
+    vexist=$(sudo curve list --user root --dirname / --password root_password | grep -E "^${filename}$")
+    if [ "$vexist" ]
     then
-        echo "CURVE volume info: filename: ${filename}, filelength: ${filelength}, user: ${user}"
-        sudo curve create --filename /${filename} --length ${filelength} --user ${user}
+        filepath=/${filename}
+        echo "CURVE volume /${filename} exists"
     else
-        echo "CURVE volume ${filename} exists"
+        # check user dir exist in root dir
+        userdir=$(sudo curve list --user root --dirname / --password root_password | grep -E "^${user}$")
+        if [ ! "$userdir" ]
+        then
+            sudo curve mkdir --user ${user} --dirname /${user}
+        else
+            echo "CURVE user dir /${user} exists"
+        fi
+        # check the user dir create success or not
+        userdir=$(sudo curve list --user root --dirname / --password root_password | grep -E "^${user}$")
+        if [ ! "$userdir" ]
+        then
+            echo "ERROR: create CURVE user dir error!"
+            exit
+        else
+            # check volume exist in user dir
+            volume=$(sudo curve list --user ${user} --dirname /${user} | grep -E "^${filename}$")
+            if [ ! "$volume" ]
+            then
+                echo "CURVE volume info: filename: /${user}/${filename}, filelength: ${filelength}, user: ${user}"
+                sudo curve create --filename /${user}/${filename} --length ${filelength} --user ${user}
+            else
+                echo "CURVE volume /${user}/${filename} exists"
+            fi
+            filepath=/${user}/${filename}
+        fi
     fi
 
     # map curve volume to nbd device
     echo "2. map CURVE volume to nds device"
-    volume=$(sudo curve_ops_tool list --fileName=/ | grep ${filename})
-    nbddevice=$(sudo curve-nbd list-mapped | grep ${filename})
-    if [ ! "$volume" ]
+    volume=$(sudo curve list --user ${user} --dirname /${user} | grep -E "^${filename}$")
+    if [ ! "$vexist" -a ! "$volume" ]
     then
         echo "ERROR: create CURVE volume error!"
         exit
-    elif [ ! "$nbddevice" ]
-    then
-        echo "curve-nbd map cbd:pool//${filename}_${user}_"
-        sudo curve-nbd map cbd:pool//${filename}_${user}_
+    else
+        nbddevice=$(sudo curve-nbd list-mapped | grep ${filepath})
+        if [ ! "$nbddevice" ]
+        then
+            echo "curve-nbd map cbd:pool/${filepath}_${user}_"
+            sudo curve-nbd map cbd:pool/${filepath}_${user}_
+        else
+            echo "CURVE volume ${filepath} has mapped"
+        fi
     fi
 
     # format the nbd device to ext4 filesystem
     echo "3. format the nbd device to ext4 filesystem"
-    nbddevice=$(sudo curve-nbd list-mapped | grep ${filename} | cut -d " " -f 3)
-    fstype=$(sudo blkid -s TYPE ${nbddevice} | awk -F '"' '{print $2}')
+    nbddevice=$(sudo curve-nbd list-mapped | grep ${filepath} | cut -d " " -f 3)
     if [ ! "$nbddevice" ]
     then
         echo "ERROR: map curve volume to nbd device error!"
         exit
-    elif [ "$fstype" != "ext4" ]
-    then
-        echo "format ${nbddevice} to ext4"
-        sudo mkfs.ext4 ${nbddevice}
+    else
+        fstype=$(sudo blkid -s TYPE ${nbddevice} | awk -F '"' '{print $2}')
+        if [ "$fstype" != "ext4" ]
+        then
+            echo "format ${nbddevice} to ext4"
+            sudo mkfs.ext4 ${nbddevice}
+        else
+            echo "nbd device ${nbddevice} has been formatted"
+        fi
         sed -i "/^What=*/cWhat=$nbddevice" /etc/systemd/system/data.mount
     fi
 }

k8s/nbd/nbd-package/usr/bin/umount_curve_clouddisk.sh

@@ -19,6 +19,62 @@ function usage() {
     echo "  ./umount_curve_clouddisk.sh --filename curvefile --user k8s //use specify OPTIONS"
 }
 
+function umount() {
+    sudo umount $nbddevice
+    hasmount=$(df -T | grep ${nbddevice})
+    if [ "$hasmount" ]
+    then
+        echo "ERROR: umount nbddevice failed!"
+        exit
+    else
+        echo "umount nbddevice success"
+    fi
+}
+
+function unmap() {
+    sudo curve-nbd unmap $nbddevice
+    cnt=$(ps -ef | grep -E "curve-nbd map cbd:pool/${filepath}" | grep -v grep | wc -l)
+    while [ ! "$cnt" -eq 0 ]
+    do
+        sleep 0.5
+        cnt=$(ps -ef | grep -E "curve-nbd map cbd:pool/${filepath}" | grep -v grep | wc -l)
+    done
+
+    nbddevice=$(sudo curve-nbd list-mapped | grep ${filepath} | cut -d " " -f 3)
+    if [ "$nbddevice" ]
+    then
+        echo "ERROR: unmap nbddevice failed!"
+        exit
+    else
+        echo "unmap nbddevice success"
+    fi
+}
+
+function delvolume() {
+    read -r -p "Delete curvevolume filename=${filepath}, user=${user}. Are You Sure? [Y/n] " input
+    case $input in
+        [yY][eE][sS]|[yY])
+            sudo curve delete --filename $filepath --user $user
+            curvevolume=$(sudo curve list --user ${user} --dirname /${user} | grep -E "^${filename}$")
+            vexist=$(sudo curve list --user root --dirname / --password root_password | grep -E "^${filename}$")
+            if [ "$curvevolume" -o "$vexist" ]
+            then
+                echo "ERROR: delete curve volume failed!"
+                exit
+            else
+                echo "delete curve volume success"
+            fi
+            ;;
+        [nN][oO]|[nN])
+            exit 1
+            ;;
+        *)
+        echo "Invalid input..."
+        exit 1
+        ;;
+    esac
+}
+
 function run() {
     # check required OPTIONS
     if [ ! "$filename" -o ! "$user" ]
@@ -28,57 +84,43 @@ function run() {
         exit
     fi
 
-    curvevolume=$(sudo curve_ops_tool list --fileName=/ | grep ${filename})
-    nbddevice=$(sudo curve-nbd list-mapped | grep ${filename} | cut -d " " -f 3)
-    hasmount=$(df -T | grep ${nbddevice})
-
-    if [ "$hasmount" ]
+    curvevolume=$(sudo curve list --user ${user} --dirname /${user} | grep -E "^${filename}$")
+    vexist=$(sudo curve list --user root --dirname / --password root_password | grep -E "^${filename}$")
+    if [ ! "$curvevolume" -a ! "$vexist" ]
     then
-        sudo umount $nbddevice
-        hasmount=$(df -T | grep ${nbddevice})
-        if [ "$hasmount" ]
+        echo "clean success"
+        exit
+    elif [ "$curvevolume" -a "$vexist" ]
+    then
+        echo "ERROR: volume: ${filename} exist both in root dir and /${user} dir"
+        exit
+    else
+        if [ "$vexist" ]
         then
-            echo "ERROR: umount nbddevice failed!"
+            filepath=/${filename}
         else
-            echo "umount nbddevice success"
+            filepath=/${user}/${filename}
         fi
-    fi
 
-    if [ "$nbddevice" ]
-    then
-        sudo curve-nbd unmap $nbddevice
-        nbddevice=$(sudo curve-nbd list-mapped | grep ${filename} | cut -d " " -f 3)
-        if [ "$nbddevice" ]
+        nbddevice=$(sudo curve-nbd list-mapped | grep ${filepath} | cut -d " " -f 3)
+        if [ ! "$nbddevice" ]
         then
-            echo "ERROR: unmap nbddevice failed!"
+            delvolume
         else
-            echo "unmap nbddevice success"
+            hasmount=$(df -T | grep ${nbddevice})
+            if [ ! "$hasmount" ]
+            then
+                unmap
+                delvolume
+                echo "clean success"
+            else
+                umount
+                unmap
+                delvolume
+                echo "clean success"
+            fi
         fi
     fi
-
-    if [ "$curvevolume" ]
-    then
-        read -r -p "Delete curvevolume filename=${filename}, user=${user}. Are You Sure? [Y/n] " input
-        case $input in
-            [yY][eE][sS]|[yY])
-                sudo curve delete --filename /$filename --user $user
-                curvevolume=$(sudo curve_ops_tool list --fileName=/ | grep ${filename})
-                if [ "$curvevolume" ]
-                then
-                    echo "ERROR: delete curve volume failed!"
-                else
-                    echo "delete curve volume success"
-                fi
-                ;;
-            [nN][oO]|[nN])
-                exit 1
-                ;;
-            *)
-            echo "Invalid input..."
-            exit 1
-            ;;
-        esac
-    fi
 }
 
 # get required OPTIONS from confPath

k8s/nebd/nebd-package/DEBIAN/postinst

@@ -12,4 +12,4 @@ systemctl daemon-reload
 systemctl enable nebd-daemon.service
 systemctl start nebd-daemon.service
 
-exit 0
+exit 0

k8s/nebd/nebd-package/DEBIAN/preinst

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+mkdir -p /var/lib/nebd/lock
+mkdir -p /var/log/nebd/client
+mkdir -p /var/log/nebd/server
+mkdir -p /var/log/curve
+
+chown -R root:root /var/log/curve
+chmod -R 777 /var/log/curve
+
+exit 0

k8s/nebd/nebd-package/etc/nebd/nebd-client.conf

@@ -0,0 +1,28 @@
+# part2 socket file address
+nebdserver.serverAddress=/var/lib/nebd/nebd.sock
+
+# 文件锁路径
+metacache.fileLockPath=/var/lib/nebd/lock
+
+# 同步rpc的最大重试次数
+request.syncRpcMaxRetryTimes=50
+# rpc请求的重试间隔
+request.rpcRetryIntervalUs=100000
+# rpc请求的最大重试间隔
+request.rpcRetryMaxIntervalUs=64000000
+# rpc hostdown情况下的重试时间
+request.rpcHostDownRetryIntervalUs=10000
+# brpc的健康检查周期时间，单位s
+request.rpcHealthCheckIntervalS=1
+# brpc从rpc失败到进行健康检查的最大时间间隔，单位ms
+request.rpcMaxDelayHealthCheckIntervalMs=100
+# rpc发送执行队列个数
+request.rpcSendExecQueueNum=2
+
+# heartbeat间隔
+heartbeat.intervalS=5
+# heartbeat rpc超时时间
+heartbeat.rpcTimeoutMs=500
+
+# 日志路径
+log.path=/var/log/nebd/client

k8s/nebd/nebd-package/etc/nebd/nebd-server.conf

@@ -0,0 +1,14 @@
+# curve-client配置文件地址
+curveclient.confPath=/etc/curve/client.conf
+
+#brpc server监听端口
+listen.address=/var/lib/nebd/nebd.sock
+
+#元数据文件地址,包含文件名
+meta.file.path=/var/lib/nebd/nebdserver.meta
+
+#心跳超时时间
+heartbeat.timeout.sec=30
+
+#文件超时检测时间间隔
+heartbeat.check.interval.ms=3000

mk-deb.sh

@@ -528,6 +528,7 @@ cp -r nebd/nebd-package build/
 mkdir -p build/nebd-package/usr/bin
 mkdir -p build/nebd-package/usr/lib/nebd
 
+mkdir -p k8s/nebd/nebd-package/usr/bin
 cp nebd/nebd-package/usr/bin/nebd-daemon k8s/nebd/nebd-package/usr/bin
 sed -i '/^baseLogPath=/cbaseLogPath=/var/log/nebd' k8s/nebd/nebd-package/usr/bin/nebd-daemon
 cp -r k8s/nebd/nebd-package build/k8s-nebd-package