fix: ansible deploy mutiple machine

Author: bai-charisu

conf/chunkserver.conf.example

@@ -107,9 +107,9 @@ clone.thread_num=10
 # 克隆的队列深度
 clone.queue_depth=6000
 # curve用户名
-curve.root_username=root
+curve.root_username=xxx
 # curve密码
-curve.root_password=root_password
+curve.root_password=xxx
 # client配置文件
 curve.config_path=conf/cs_client.conf
 # s3配置文件

conf/mds.conf

@@ -113,8 +113,8 @@ mds.file.scanIntevalTimeUs=500000
 # auth settings
 #
 # root用户密码
-mds.auth.rootUserName=root
-mds.auth.rootPassword=root_password
+mds.auth.rootUserName=xxx
+mds.auth.rootPassword=xxx
 
 #
 # file lock setting

conf/snapshot_clone_server.conf

@@ -4,9 +4,9 @@
 # client配置文件位置
 client.config_path=conf/cs_client.conf
 # mds root 用户名
-mds.rootUser=root
+mds.rootUser=xxx
 # mds root 密码
-mds.rootPassword=root_password
+mds.rootPassword=xxx
 # 调用client方法的重试总时间
 client.methodRetryTimeSec=300
 # 调用client方法重试间隔时间

curve-ansible/common_tasks/wait_until_server_down.yml

@@ -17,6 +17,7 @@
 
 - name: wait until server is down
   wait_for:
+    host: "{{ ansible_ssh_host }}"
     port: "{{ ping_port }}"
     state: stopped
     msg: "the server {{ ansible_host }}:{{ ping_port }} is down"

curve-ansible/common_tasks/wait_until_server_up.yml

@@ -17,6 +17,7 @@
 
 - name: wait until server is up
   wait_for:
+    host: "{{ ansible_ssh_host }}"
     port: "{{ ping_port }}"
     state: started
     timeout: "{{ wait_service_timeout }}"

curve-ansible/deploy_snapshotcloneserver.yml

@@ -27,7 +27,7 @@
   tags:
     - prepare
   roles:
-    - { role: install_package, package_name: snapshotcloneserver, package_version: "{{ snapshot_package_version }}",
+    - { role: install_package, package_name: curve-snapshotcloneserver, package_version: "{{ snapshot_package_version }}",
             tags: ["install_package", "install_snap_package"] }
     - { role: install_package, package_name: aws-sdk, package_version: "{{ aws_package_version }}",
             tags: ["install_package", "install_aws_sdk_package"] }

curve-ansible/deploy_snapshotcloneserver_nginx.yml

@@ -27,7 +27,7 @@
   tags:
     - prepare
   roles:
-    - { role: install_package, package_name: snapshotcloneserver_nginx,
+    - { role: install_package, package_name: curve-snapshotcloneserver-nginx,
             tags: ["install_conf_package", "install_snapshotclone_nginx_conf_package"] }
     - { role: generate_config, template_name: nginx.conf, conf_path: "{{ snapshot_nginx_conf_path }}",
             tags: ["generate_config", "generate_snap_nginx_conf"] }

curve-ansible/roles/generate_config/defaults/main.yml

@@ -16,8 +16,8 @@
 #
 
 # 通用配置
-curve_root_username: root
-curve_root_password: root_password
+curve_root_username: xxx
+curve_root_password: xxx
 chunk_size: 16777216
 file_expired_time_us: 5000000
 
@@ -104,8 +104,6 @@ chunkserver_copyset_load_concurrency: 10
 chunkserver_copyset_check_retrytimes: 3
 chunkserver_copyset_finishload_margin: 2000
 chunkserver_copyset_check_loadmargin_interval_ms: 1000
-chunkserver_clone_disable_curve_client: false
-chunkserver_clone_disable_s3_adapter: false
 chunkserver_clone_slice_size: 1048576
 chunkserver_clone_enable_paste: false
 chunkserver_clone_thread_num: 10

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -100,9 +100,9 @@ copyset.check_loadmargin_interval_ms={{ chunkserver_copyset_check_loadmargin_int
 # Clone settings
 #
 # 禁止使用curveclient
-clone.disable_curve_client={{ chunkserver_clone_disable_curve_client }}
+clone.disable_curve_client={{ disable_snapshot_clone }}
 # 禁止使用s3adapter
-clone.disable_s3_adapter={{ chunkserver_clone_disable_s3_adapter }}
+clone.disable_s3_adapter={{ disable_snapshot_clone }}
 # 克隆的分片大小，一般1MB
 clone.slice_size={{ chunkserver_clone_slice_size }}
 # 读clone chunk时是否需要paste到本地

curve-ansible/roles/install_package/tasks/include/install_snapshotcloneserver_nginx.yml curve-ansible/roles/install_package/tasks/include/install_curve-snapshotcloneserver-nginx.yml

@@ -1,5 +1,6 @@
 #!/bin/bash
 #confirm提示，防止误操作
+dataDir={{ chunkserver_data_dir }}
 function do_confirm {
   echo "This deployment script will format the disk and delete all the data."
   echo "Please confirm that you want to do this action!"
@@ -19,7 +20,7 @@ function deploy_prep {
   if [ $? -eq 0 ]
   then
     sed -i '/curvefs/d' /etc/fstab
-    sed -i '/\/data\/chunkserver/d' /etc/fstab
+    sed -i '/chunkserver/d' /etc/fstab
   fi
 #通过lsscsi的输出过滤出curve使用的数据盘，将数据盘挂载的目录都卸载掉，为下一步格式化磁盘做准备
   for i in `lsscsi |grep ATA|awk '{print $7}'|awk -F"/" '{print $3}'`
@@ -41,7 +42,6 @@ function deploy_prep {
 }
 {% raw %}
 #记录磁盘的盘符信息和磁盘的wwn信息，将信息持久化到diskinfo文件
-DATA_DIR=/data
 declare -A disk_map
 diskinfo=./diskinfo
 function record_diskinfo {
@@ -61,23 +61,23 @@ function record_diskinfo {
   done
 }
 
-#根据磁盘数量创建数据目录和日志目录，目前的数据目录格式统一是/data/chunkserver+num，日志目录在/data/log/chunkserver+num
+#根据磁盘数量创建数据目录和日志目录，目前的数据目录格式统一是$dataDir/chunkserver+num，日志目录在$dataDir/log/chunkserver+num
 function chunk_dir_prep {
-  if [ -d ${DATA_DIR} ]
+  if [ -d ${dataDir} ]
   then
-    rm -rf ${DATA_DIR}
+    rm -rf ${dataDir}/chunkserver*
     if [ $? -ne 0 ]
     then
-    echo "rm $DATA_DIR failed"
+    echo "rm $dataDir failed"
     exit
     fi
   fi
-  mkdir -p ${DATA_DIR}
+  mkdir -p ${dataDir}
   echo $((${#disk_map[@]}-1))
   for i in `seq 0 $((${#disk_map[@]}-1))`
   do
-    mkdir -p ${DATA_DIR}/chunkserver$i
-    mkdir -p ${DATA_DIR}/log/chunkserver$i
+    mkdir -p ${dataDir}/chunkserver$i
+    mkdir -p ${dataDir}/log/chunkserver$i
   done
 }
 {% endraw %}
@@ -107,7 +107,7 @@ function mount_dir {
   j=0
   for i in `cat $diskinfo |awk '{print $1}'|awk -F":" '{print $2}'`
   do
-  mount /dev/$i $DATA_DIR/chunkserver$j
+  mount /dev/$i $dataDir/chunkserver$j
     if [ $? -ne 0 ]
     then
       echo "mount $i failed"
@@ -137,7 +137,7 @@ function meta_record {
   grep curvefs /etc/fstab
   if [ $? -eq 0 ]
   then
-  for i in `cat /etc/fstab | grep "/data/chunkserver" | awk '{print $1 $2}' | awk -F '=' '{print $2}'`
+  for i in `cat /etc/fstab | grep "$dataDir/chunkserver" | awk '{print $1 $2}' | awk -F '=' '{print $2}'`
   do
     uuid=`echo $i | awk -F / '{print $1}'`
     uuidmd5=`echo -n $uuid | md5sum | cut -d ' ' -f1`
@@ -155,9 +155,9 @@ ret=`lsblk|grep chunkserver|wc -l`
 for i in `seq 0 $((${ret}-1))`
 do
   curve-format -allocatepercent={{ chunk_alloc_percent }} \
-  -chunkfilepool_dir=/data/chunkserver$i/chunkfilepool \
-  -chunkfilepool_metapath=/data/chunkserver$i/chunkfilepool.meta \
-  -filesystem_path=/data/chunkserver$i/chunkfilepool &
+  -chunkfilepool_dir=$dataDir/chunkserver$i/chunkfilepool \
+  -chunkfilepool_metapath=$dataDir/chunkserver$i/chunkfilepool.meta \
+  -filesystem_path=$dataDir/chunkserver$i/chunkfilepool &
 done
 wait
 }
@@ -168,7 +168,7 @@ function usage {
   echo "      or deploy one disk by setting the diskname"
   echo "Example:"
   echo "        ./chunkserver_deploy2.sh all"
-  echo "        ./chunkserver_deploy2.sh /dev/sd{id} /data/chunkserver{id}"
+  echo "        ./chunkserver_deploy2.sh /dev/sd{id} $dataDir/chunkserver{id}"
   exit
 }
 

curve-ansible/roles/install_package/tasks/include/install_snapshotcloneserver.yml curve-ansible/roles/install_package/tasks/include/install_curve-snapshotcloneserver.yml

@@ -18,39 +18,25 @@
 
 # 滚动升级snapshotclone
 
-# 1. 更新各节点上的配置
-- name: update config
+# 1、准备包和配置
+- name: prepare snapshotcloneserver
   hosts: snapshotclone
-  gather_facts: no
   any_errors_fatal: true
-  become: true
-  become_user: "{{ sudo_user }}"
-  become_flags: -iu {{ sudo_user }}
-  tags:
-    - update_config
-  roles:
-    - { role: generate_config, template_name: snapshot_clone_server.conf, conf_path: "{{ snapshot_config_path }}" }
-    - { role: generate_config, template_name: s3.conf, conf_path: "{{ snap_s3_config_path }}" }
-    - { role: generate_config, template_name: client.conf, conf_path: "{{ snap_client_config_path }}" }
-    - { role: generate_config, template_name: snapshot_tools.conf, conf_path: "{{ snap_tool_config_path }}" }
-
-# 2. 更新各节点上的包和配置
-- name: update package
-  hosts: snapshotclone
   gather_facts: no
-  any_errors_fatal: true
-  become: true
+  become: yes
   become_user: "{{ sudo_user }}"
   become_flags: -iu {{ sudo_user }}
   tags:
-    - update_package
-
-  tasks:
-    - vars:
-        update_component: "curve-snapshotcloneserver={{ snap_package_version }}"
-      include_tasks: "common_tasks/update_package.yml"
+    - prepare
+  roles:
+    - { role: install_package, package_name: curve-snapshotcloneserver, package_version: "{{ snapshot_package_version }}",
+            tags: ["install_package", "install_snap_package"] }
+    - { role: generate_config, template_name: snapshot_clone_server.conf, conf_path: "{{ snapshot_config_path }}",
+            tags: ["generate_config", "generate_snap_conf"] }
+    - { role: generate_config, template_name: client.conf, conf_path: "{{ snap_client_config_path }}",
+            tags: ["generate_config", "generate_snap_client_conf"] }
 
-# 3. 获取leader节点和follower节点
+# 2. 获取leader节点和follower节点
 - name: set leader and follower list
   hosts: snapshotclone
   gather_facts: no
@@ -60,7 +46,7 @@
   roles:
     - { role: set_leader_and_follower_list, service_name: snapshot }
 
-# 4. 按顺序先升级follower节点，再升级leader节点
+# 3. 按顺序先升级follower节点，再升级leader节点
 - name: update follower and leader server in sequence
   hosts: snapshot_servers_followers, snapshot_servers_leader
   any_errors_fatal: true

curve-ansible/roles/install_package/templates/chunkserver_deploy.sh.j2

@@ -3,7 +3,7 @@
 localhost ansible_ssh_host=127.0.0.1
 
 [etcd]
-localhost ansible_ssh_host=127.0.0.1
+localhost ansible_ssh_host=127.0.0.1 etcd_name=etcd1
 
 [snapshotclone]
 localhost ansible_ssh_host=127.0.0.1
@@ -15,7 +15,6 @@ localhost ansible_ssh_host=127.0.0.1
 localhost ansible_ssh_host=127.0.0.1
 
 [mds:vars]
-ansible_ssh_port=1046
 mds_dummy_port=6667
 mds_port=6666
 mds_subnet=127.0.0.1/22
@@ -30,7 +29,6 @@ mds_log_dir=/data/log/curve/mds
 topo_file_path=/etc/curve/topo.json
 
 [etcd:vars]
-ansible_ssh_port=1046
 etcd_listen_client_port=2379
 etcd_listen_peer_port=2380
 etcd_name="etcd"
@@ -40,13 +38,12 @@ etcd_config_path=/etc/curve/etcd.conf.yml
 etcd_log_dir=/data/log/curve/etcd
 
 [snapshotclone:vars]
-ansible_ssh_port=1046
 snapshot_port=5556
 snapshot_dummy_port=8081
 snapshot_subnet=127.0.0.1/22
 defined_healthy_status="cluster is healthy"
 snapshot_package_version="0.0.6.1.1+7af4d6a4"
-snapshot_need_sudo=false
+snapshot_need_sudo=true
 snapshot_config_path=/etc/curve/snapshot_clone_server.conf
 snap_s3_config_path=/etc/curve/s3.conf
 snap_client_config_path=/etc/curve/snap_client.conf
@@ -57,13 +54,10 @@ client_chunkserver_max_rpc_timeout_ms=16000
 client_chunkserver_max_stable_timeout_times=64
 client_turn_off_health_check=false
 snapshot_clone_server_log_dir=/data/log/curve/snapshotclone
-s3_ak=""
-s3_sk=""
 aws_package_version="1.0"
 
 [chunkservers:vars]
 wait_service_timeout=60
-ansible_ssh_port=1046
 env_name=pubt1
 check_copysets_status_times=1000
 check_copysets_status_interval=1
@@ -72,6 +66,7 @@ aws_package_version="1.0"
 defined_copysets_status="Copysets are healthy"
 chunkserver_base_port=8200
 chunkserver_format_disk=false
+chunk_alloc_percent=80
 # 每台机器上的chunkserver的数量
 chunkserver_num=3
 chunkserver_need_sudo=true
@@ -90,6 +85,7 @@ client_register_to_mds=false
 client_chunkserver_op_max_retry=3
 client_chunkserver_max_stable_timeout_times=64
 client_turn_off_health_check=false
+disable_snapshot_clone=true
 
 [snapshotclone_nginx:vars]
 snapshotcloneserver_nginx_dir=/etc/curve/nginx
@@ -102,7 +98,11 @@ nginx_docker_external_port=5555
 need_confirm=true
 curve_ops_tool_config=/etc/curve/tools.conf
 need_update_config=true
-mysql_url=localhost
 wait_service_timeout=10
 sudo_user=curve
 deploy_dir=/home/curve
+s3_ak=""
+s3_sk=""
+ansible_ssh_port=22
+curve_root_username=xxx
+curve_root_password=xxx

curve-ansible/rolling_update_snapshotclone.yml

@@ -36,7 +36,7 @@ ansible是一款自动化运维工具，curve-ansible 是基于 ansible playbook
 
 1. 以 ```root``` 用户登录中控机， 在中控机上创建 ```curve``` 用户， 并生成ssh key
 
-2. 下载tar包 并 解压
+2. 下载tar包并解压
 
    ```
    wget https://github.com/opencurve/curve/releases/download/v0.1.0/curve_0.1.0+0bd75f0a.tar.gz

curve-ansible/server.ini

@@ -7674,9 +7674,9 @@
       {
         "allValue": "",
         "current": {
-          "text": "pubbeta2-curve22.dg.163.org",
+          "text": "localhost",
           "value": [
-            "pubbeta2-curve22.dg.163.org"
+            "localhost"
           ]
         },
         "datasource": "Prometheus",