Merge remote-tracking branch 'origin/develop-1.1.1' into feature-1.2-…

…fate-flow-push-model # Conflicts: # fate_flow/manager/tracking.py
RainMan1860 · Nov 27, 2019 · 0cb1b30 · 0cb1b30
2 parents 76703e6 + 4ed76f0
commit 0cb1b30
Show file tree

Hide file tree

Showing 60 changed files with 668 additions and 457 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,8 +1,8 @@
 [submodule "fateboard"]
 	path = fateboard
 	url = https://github.com/FederatedAI/FATE-Board.git
-	branch = v1.1
+	branch = develop-1.1.1
 [submodule "eggroll"]
 	path = eggroll
 	url = https://github.com/WeBankFinTech/eggroll.git
-	branch = v1.1.1
+	branch = release-1.1.2-build3
diff --git a/README.md b/README.md
@@ -13,11 +13,11 @@ FATE (Federated AI Technology Enabler) is an open-source project initiated by We
 
 *  Join our maillist [Fate-FedAI Group IO](https://groups.io/g/Fate-FedAI). You can ask questions and participate in the development discussion.
 
-*  For any frequently asked questions, you can check in [FAQ](https://github.com/WeBankFinTech/FATE/wiki).  
+*  For any frequently asked questions, you can check in [FAQ](https://github.com/FederatedAI/FATE/wiki/FATE-FAQ).  
 
-*  Please report bugs by submitting [issues](https://github.com/WeBankFinTech/FATE/issues). 
+*  Please report bugs by submitting [issues](https://github.com/FederatedAI/FATE/issues). 
 
-*  Submit contributions using [pull requests](https://github.com/WeBankFinTech/FATE/pulls)
+*  Submit contributions using [pull requests](https://github.com/FederatedAI/FATE/pulls)
 
 
 ## Federated Learning Algorithms In FATE

diff --git a/arch/api/table/eggroll_util.py b/arch/api/table/eggroll_util.py
@@ -42,7 +42,8 @@ def build_eggroll_runtime(work_mode: WorkMode, eggroll_session: EggrollSession):
         from eggroll.api.cluster.eggroll import eggroll_init, _EggRoll
         if _EggRoll.instance is None:
             return eggroll_init(eggroll_session)
-    raise ValueError(f"work_mode: {work_mode} not supported!")
+    else:
+        raise ValueError(f"work_mode: {work_mode} not supported!")
 
 
 def broadcast_eggroll_session(sc, work_mode, eggroll_session):

diff --git a/arch/api/table/pyspark/rdd_func.py b/arch/api/table/pyspark/rdd_func.py
@@ -28,7 +28,7 @@ def _save_as_func(rdd: RDD, name, namespace, partition, persistent):
 
     def _func(_, it):
         eggroll_util.maybe_create_eggroll_client()
-        dup.put_all(it)
+        dup.put_all(list(it))
         return 1,
 
     rdd.mapPartitionsWithIndex(_func, preservesPartitioning=False).collect()

diff --git a/cluster-deploy/README.md b/cluster-deploy/README.md
@@ -79,15 +79,24 @@ c).Other Modules: The node where other modules are installed.
 
 *<u>Note: The above nodes can be the same node, but to distinguish the description, the following is explained in the case of full distribution.</u>*
 
-| node           | Node description                                  | Software configuration                                       | Software installation path                                   | Network Configuration                                        |
-| -------------- | ------------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ |
-| Execution node | The operation node that executes the script       | Git tool   rsync Maven 3.5 and above                         | Install it using the yum install command.                    | Interworking with the public network, you can log in to other node app users without encryption. |
-| Meta-Service   | The node where the meta service module is located | Jdk1.8+       Python3.6  python virtualenv mysql8.0.13       | /data/projects/common/jdk/jdk1.8  /data/projects/common/miniconda3  /data/projects/fate/venv  /data/projects/common/mysql/mysql-8.0.13 | In the same area or under the same VPC as other nodes        |
-| Other Modules  | Node where other modules are located              | Jdk1.8+  Python3.6  python virtualenv redis5.0.2(One party only needs to install a redis on the serving-service node.) | /data/projects/common/jdk/jdk1.8 /data/projects/common/miniconda3 /data/projects/fate/venv /data/projects/common/redis/redis-5.0.2 | In the same area or under the same VPC as other nodes.       |
+| node           | Node description                            | Software configuration                                       | Software installation path                                   | Network Configuration                                        |
+| -------------- | ------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| Execution node | The operation node that executes the script | Git tool   rsync Maven 3.5 and above                         | Install it using the yum install command.                    | Interworking with the public network, you can log in to other node app users without encryption. |
+| Modules        | Node where modules are located              | Jdk1.8+  Python3.6  python virtualenv redis5.0.2(One party only needs to install a redis on the serving-service node.)      mysql8.0.13(One party only needs to install a redis on the Meta-Service node.) | /data/projects/common/jdk/jdk1.8 /data/projects/common/miniconda3 /data/projects/fate/venv /data/projects/common/redis/redis-5.0.2                                                                   /data/projects/common/mysql/mysql-8.0.13 | In the same area or under the same VPC as other nodes.       |
 
 Check whether the above software environment is reasonable in the corresponding server. If the software environment already exists and the correct installation path corresponds to the above list, you can skip this step. If not, refer to the following initialization steps to initialize the environment:
 
-### 3.3.**Configuring sudo**
+### 3.3.**create user**
+
+Executed by the **root** user:
+
+```
+groupadd -g 6000 apps
+useradd -s /bin/bash -g apps -d /home/app app
+passwd app
+```
+
+### 3.4.**Configuring sudo**
 
 Executed by the **root** user:
 
@@ -101,7 +110,7 @@ app ALL=(ALL) NOPASSWD: ALL
 
 Defaults !env_reset
 
-### 3.6.Configure password-free login from the execution node to the node to be deployed
+### 3.5.Configure password-free login from the execution node to the node to be deployed
 
 1.Generate a key on the all node,Contains the execution node and the node to be deployed
 

diff --git a/cluster-deploy/doc/Fate-V1.1allinone部署指南(build).md b/cluster-deploy/doc/Fate-V1.1allinone部署指南(build).md
@@ -80,6 +80,16 @@ systemctl status firewalld.service
 3.5 软件环境初始化
 ------------------
 
+**1）创建用户**
+
+**在目标服务器（192.168.0.1 192.168.0.2）root用户下执行**
+
+```
+groupadd -g 6000 apps
+useradd -s /bin/bash -g apps -d /home/app app
+passwd app
+```
+
 **1）配置sudo**
 
 **在目标服务器（192.168.0.1 192.168.0.2）root用户下执行**
@@ -185,21 +195,7 @@ redis_password=fate_dev
 cxx_compile_flag=false
 ```
 
-**2）一台主机partyA+partyB同时部署****
-
-```
-#!/bin/bash
-
-user=app
-deploy_dir=/data/projects/fate
-party_list=(10000 9999)
-node_list=(192.168.0.1)
-db_auth=(fate_dev fate_dev)
-redis_password=fate_dev
-cxx_compile_flag=false
-```
-
-**3）只部署一个party**
+**2）只部署一个party**
 
 ```
 #!/bin/bash

diff --git a/cluster-deploy/doc/Fate-V1.1allinone部署指南(install).md b/cluster-deploy/doc/Fate-V1.1allinone部署指南(install).md
@@ -80,7 +80,17 @@ systemctl status firewalld.service
 3.5 软件环境初始化
 ------------------
 
-**1）配置sudo**
+**1）创建用户**
+
+**在目标服务器（192.168.0.1 192.168.0.2）root用户下执行**
+
+```
+groupadd -g 6000 apps
+useradd -s /bin/bash -g apps -d /home/app app
+passwd app
+```
+
+**2）配置sudo**
 
 **在目标服务器（192.168.0.1 192.168.0.2）root用户下执行**
 
@@ -92,7 +102,7 @@ app ALL=(ALL) NOPASSWD: ALL
 
 Defaults !env_reset
 
-**2）配置ssh无密登录**
+**3）配置ssh无密登录**
 
 **a. 在目标服务器（192.168.0.1 192.168.0.2）app用户下执行**
 
@@ -180,22 +190,7 @@ redis_password=fate_dev
 cxx_compile_flag=false
 ```
 
-**2）一台主机partyA+partyB同时部署****
-
-```
-#!/bin/bash
-
-user=app
-
-deploy_dir=/data/projects/fate
-party_list=(10000 9999)
-node_list=(192.168.0.1)
-db_auth=(fate_dev fate_dev)
-redis_password=fate_dev
-cxx_compile_flag=false
-```
-
-**3）只部署一个party**
+**2）只部署一个party**
 
 ```
 #!/bin/bash

diff --git a/cluster-deploy/doc/Fate-V1.1cluster部署指南(build).md b/cluster-deploy/doc/Fate-V1.1cluster部署指南(build).md
@@ -80,7 +80,17 @@ systemctl status firewalld.service
 3.5 软件环境初始化
 ------------------
 
-**1）配置sudo**
+**1）创建用户**
+
+**在目标服务器（192.168.0.1 192.168.0.2）root用户下执行**
+
+```
+groupadd -g 6000 apps
+useradd -s /bin/bash -g apps -d /home/app app
+passwd app
+```
+
+**2）配置sudo**
 
 **在目标服务器（192.168.0.1 192.168.0.2）root用户下执行**
 
@@ -92,7 +102,7 @@ app ALL=(ALL) NOPASSWD: ALL
 
 Defaults !env_reset
 
-**2）配置ssh无密登录**
+**3）配置ssh无密登录**
 
 **a. 在目标服务器（192.168.0.1 192.168.0.2）app用户下执行**
 

diff --git a/cluster-deploy/doc/Fate-V1.1cluster部署指南(install).md b/cluster-deploy/doc/Fate-V1.1cluster部署指南(install).md
@@ -80,7 +80,17 @@ systemctl status firewalld.service
 3.5 软件环境初始化
 ------------------
 
-**1）配置sudo**
+**1）创建用户**
+
+**在目标服务器（192.168.0.1 192.168.0.2）root用户下执行**
+
+```
+groupadd -g 6000 apps
+useradd -s /bin/bash -g apps -d /home/app app
+passwd app
+```
+
+**2）配置sudo**
 
 **在目标服务器（192.168.0.1 192.168.0.2）root用户下执行**
 
@@ -92,7 +102,7 @@ app ALL=(ALL) NOPASSWD: ALL
 
 Defaults !env_reset
 
-**2）配置ssh无密登录**
+**3）配置ssh无密登录**
 
 **a. 在目标服务器（192.168.0.1 192.168.0.2）app用户下执行**
 

diff --git a/cluster-deploy/scripts/default_configurations.sh b/cluster-deploy/scripts/default_configurations.sh
@@ -1,5 +1,13 @@
 #!/bin/bash
 fate_cos_address=https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com
+version=1.1.1
+egg_version=1.1
+meta_service_version=1.1
+roll_version=1.1
+federation_version=1.1
+proxy_version=1.1
+fateboard_version=1.1
+fateflow_version=1.1
 python_version=1.1
 jdk_version=8u192
 mysql_version=8.0.13

diff --git a/cluster-deploy/scripts/deploy/eggroll/egg/configurations.sh b/cluster-deploy/scripts/deploy/eggroll/egg/configurations.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 #federation
 
-version=1.1
+version=
+egg_version=
 java_dir=
 source_code_dir=
 output_packages_dir=
@@ -19,3 +20,4 @@ roll_ip=
 roll_port=8011
 proxy_ip=
 proxy_port=9370
+clustercomm_ip=
diff --git a/cluster-deploy/scripts/deploy/eggroll/egg/deploy.sh b/cluster-deploy/scripts/deploy/eggroll/egg/deploy.sh
@@ -146,12 +146,29 @@ install(){
     cp -r ${deploy_packages_dir}/source/${module_name}/egg-manager/* ${deploy_dir}/${module_name}/
     cp -r ${deploy_packages_dir}/config/${module_name}/conf/* ${deploy_dir}/${module_name}/
     cd ${deploy_dir}/${module_name}
-    ln -s eggroll-${module_name}-${version}.jar eggroll-${module_name}.jar
+    ln -s eggroll-${module_name}-${egg_version}.jar eggroll-${module_name}.jar
     mv ./services.sh ${deploy_dir}/
 
     cd ${deploy_packages_dir}/source/${module_name}/egg-services
 
     mkdir -p ${deploy_dir}/storage-service-cxx
+
+    system=`sed -e '/"/s/"//g' /etc/os-release | awk -F= '/^NAME/{print $2}'`
+    echo ${system}
+    case "${system}" in
+        "CentOS Linux")
+                echo "CentOS System"
+                rm -rf ./storage-service-cxx/third_party_eggrollv1_ubuntu
+                ;;
+        "Ubuntu")
+                echo "Ubuntu System"
+                rm -rf ./storage-service-cxx/third_party
+                mv ./storage-service-cxx/third_party_eggrollv1_ubuntu  ./storage-service-cxx/third_party
+                ;;
+        *)
+                echo "Not support this system."
+    esac
+
     cp -r ./storage-service-cxx/* ${deploy_dir}/storage-service-cxx/
 
     mkdir -p ${deploy_dir}/python/eggroll/computing
@@ -163,6 +180,17 @@ install(){
     mkdir -p ${deploy_dir}/python/eggroll/conf
     cp -r ./eggroll-conf/* ${deploy_dir}/python/eggroll/conf/
 
+    system=`sed -e '/"/s/"//g' /etc/os-release | awk -F= '/^NAME/{print $2}'`
+    echo ${system}
+    pwd
+    if [[ "${system}" == "Ubuntu" ]];then
+        cd ${deploy_dir}/storage-service-cxx/third_party
+        cd rocksdb
+        sudo cp librocksdb.so /usr/local/lib
+        sudo mkdir -p /usr/local/include/rocksdb/
+        sudo cp -r ./include/* /usr/local/include/
+    fi
+
     cd ${deploy_dir}/storage-service-cxx
 	sed -i.bak "20s#-I. -I.*#-I. -I${deploy_dir}/storage-service-cxx/third_party/include#g" ./Makefile
 	sed -i.bak "34s#LDFLAGS += -L.*#LDFLAGS += -L${deploy_dir}/storage-service-cxx/third_party/lib -llmdb -lboost_system -lboost_filesystem -lglog -lgpr#g" ./Makefile
@@ -176,6 +204,7 @@ install(){
 	sed -i.bak "s/rollport=.*/rollport=${roll_port}/g" ./modify_json.py
 	sed -i.bak "s/proxyip=.*/proxyip=\"${proxy_ip}\"/g" ./modify_json.py
 	sed -i.bak "s/proxyport=.*/proxyport=${proxy_port}/g" ./modify_json.py
+	sed -i.bak "s/clustercommip=.*/clustercommip=\"${clustercomm_ip}\"/g" ./modify_json.py
 	python ./modify_json.py python ./server_conf.json
 }
 
@@ -195,4 +224,4 @@ case "$2" in
  *)
      usage
         exit -1
-esac
+esac
diff --git a/cluster-deploy/scripts/deploy/eggroll/meta-service/configurations.sh b/cluster-deploy/scripts/deploy/eggroll/meta-service/configurations.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 #federation
 
-version=1.1
+version=
+meta_service_version=
 java_dir=
 source_code_dir=
 output_packages_dir=

diff --git a/cluster-deploy/scripts/deploy/eggroll/meta-service/deploy.sh b/cluster-deploy/scripts/deploy/eggroll/meta-service/deploy.sh
@@ -79,7 +79,7 @@ install(){
     cp -r ${deploy_packages_dir}/source/${module_name} ${deploy_dir}/
     cp -r ${deploy_packages_dir}/config/${module_name}/conf/* ${deploy_dir}/${module_name}
     cd ${deploy_dir}/${module_name}
-    ln -s eggroll-${module_name}-${version}.jar eggroll-${module_name}.jar
+    ln -s eggroll-${module_name}-${meta_service_version}.jar eggroll-${module_name}.jar
     mv ./services.sh ${deploy_dir}/
 }
 
@@ -99,4 +99,4 @@ case "$2" in
  *)
      usage
         exit -1
-esac
+esac
diff --git a/cluster-deploy/scripts/deploy/eggroll/roll/configurations.sh b/cluster-deploy/scripts/deploy/eggroll/roll/configurations.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 #federation
 
-version=1.1
+version=
+roll_version=
 java_dir=
 source_code_dir=
 output_packages_dir=

diff --git a/cluster-deploy/scripts/deploy/eggroll/roll/deploy.sh b/cluster-deploy/scripts/deploy/eggroll/roll/deploy.sh
@@ -77,7 +77,7 @@ install(){
     cp -r ${deploy_packages_dir}/source/${module_name} ${deploy_dir}/
     cp -r ${deploy_packages_dir}/config/${module_name}/conf/* ${deploy_dir}/${module_name}
     cd ${deploy_dir}/${module_name}
-    ln -s eggroll-${module_name}-${version}.jar eggroll-${module_name}.jar
+    ln -s eggroll-${module_name}-${roll_version}.jar eggroll-${module_name}.jar
     mv ./services.sh ${deploy_dir}/
 }
 
@@ -97,4 +97,4 @@ case "$2" in
  *)
      usage
         exit -1
-esac
+esac